[Bugfix][ROCm] Preserve config during GPU memory profiling (#58014)

Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
Thiago Mota Martins
2026-10-04 22:28:33 -07:00
committed by GitHub
co-authored by Fangzhou Ai
parent e8a53a55e6
commit 92044241a0
2 changed files with 45 additions and 1 deletions
+40
View File
@@ -152,6 +152,46 @@ def test_scoped_max_split_ignores_async_allocator(monkeypatch):
set_settings.assert_not_called()
@pytest.mark.parametrize("kv_cache_memory_bytes", [None, 1])
def test_memory_profile_bounds_decode_logits_rows(monkeypatch, kv_cache_memory_bytes):
"""ROCm decode workspace uses max_num_seqs during startup profiling."""
from vllm.config import get_current_vllm_config_or_none
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _max_decode_logits_rows
config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_seqs=16),
speculative_config=SimpleNamespace(num_speculative_tokens=5),
)
class Profiled(Exception):
pass
def profile_run(*, randomize_inputs=False):
assert randomize_inputs is True
# Without the config context, the helper falls back to all 32768
# batched tokens, which would reserve 128 GiB at 1M context.
assert _max_decode_logits_rows(32768) == 16 * (1 + 5)
raise Profiled
worker = SimpleNamespace(
vllm_config=config,
randomize_dummy_inputs=True,
cache_config=SimpleNamespace(kv_cache_memory_bytes=kv_cache_memory_bytes),
model_runner=SimpleNamespace(profile_run=profile_run, model_memory_usage=0),
init_snapshot=SimpleNamespace(free_memory=2),
_scoped_allocator_max_split=lambda **kwargs: nullcontext(),
)
monkeypatch.setattr(gpu_worker, "maybe_apply_startup_plan", lambda _: None)
monkeypatch.setattr(
gpu_worker, "memory_profiling", lambda *args, **kwargs: nullcontext()
)
assert get_current_vllm_config_or_none() is None
with pytest.raises(Profiled):
gpu_worker.Worker.determine_available_memory(worker)
assert get_current_vllm_config_or_none() is None
# Startup-plan persistence (vllm/v1/worker/startup_plan.py), applied and
# saved by Worker.determine_available_memory / compile_or_warm_up_model.
+5 -1
View File
@@ -615,7 +615,10 @@ class Worker(WorkerBase):
if kv_cache_memory_bytes := self.cache_config.kv_cache_memory_bytes:
# still need a profile run which compiles the model for
# max_num_batched_tokens
self.model_runner.profile_run(randomize_inputs=self.randomize_dummy_inputs)
with set_current_vllm_config(self.vllm_config):
self.model_runner.profile_run(
randomize_inputs=self.randomize_dummy_inputs
)
msg = (
f"Initial free memory {format_gib(self.init_snapshot.free_memory)} "
@@ -651,6 +654,7 @@ class Worker(WorkerBase):
# limit are never split, so they stay releasable. Exits before
# memory_profiling measures, restoring the original limit.
self._scoped_allocator_max_split(max_split_size_mb=20),
set_current_vllm_config(self.vllm_config),
):
self.model_runner.profile_run(randomize_inputs=self.randomize_dummy_inputs)