[Bugfix][CPU] Report the real cause when the worker outgrows its memory reservation (#59472)
Signed-off-by: Mo <mosafariuk@gmail.com>
This commit is contained in:
@@ -219,6 +219,7 @@ steps:
|
||||
- pytest -v -s -m 'cpu_test' v1/core
|
||||
- pytest -v -s v1/structured_output
|
||||
- pytest -v -s v1/test_serial_utils.py
|
||||
- pytest -v -s v1/worker/test_cpu_worker_kv_budget.py
|
||||
- pytest -v -s v1/test_kv_cache_spec_registry.py
|
||||
- pytest -v -s v1/cudagraph/test_cudagraph_manager.py
|
||||
- pytest -v -s -m 'cpu_test' v1/kv_connector/unit
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The CPU worker's automatic KV cache sizing must report the right cause.
|
||||
|
||||
A worker whose own footprint exceeds its reservation and a node that is short
|
||||
of free memory need opposite remedies, so they get separate errors.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.cpu_resource_utils import MemoryNodeInfo
|
||||
|
||||
# Importing the CPU worker patches torch and the GPU buffer utilities for the
|
||||
# whole process (vllm/v1/worker/cpu/shm.py), so the import is deferred into
|
||||
# the tests and they run only on the CPU platform.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cpu(), reason="CPU worker test; needs the CPU platform"
|
||||
)
|
||||
|
||||
GiB = 1 << 30
|
||||
|
||||
|
||||
def _worker(monkeypatch, *, reserved, rss, total=8 * GiB, available=4 * GiB):
|
||||
import vllm.v1.worker.cpu_worker as cpu_worker_mod
|
||||
from vllm.v1.worker.cpu_worker import CPUWorker
|
||||
|
||||
worker = CPUWorker.__new__(CPUWorker)
|
||||
worker.requested_cpu_memory = reserved
|
||||
worker.cache_config = SimpleNamespace(kv_cache_memory_bytes=None)
|
||||
monkeypatch.setattr(CPUWorker, "_should_warm_up_model", lambda self: False)
|
||||
monkeypatch.setattr(
|
||||
cpu_worker_mod,
|
||||
"get_allowed_cpu_list",
|
||||
lambda: [SimpleNamespace(numa_node=0)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
cpu_worker_mod,
|
||||
"get_memory_node_info",
|
||||
lambda node: MemoryNodeInfo(total_memory=total, available_memory=available),
|
||||
)
|
||||
process = SimpleNamespace(memory_info=lambda: SimpleNamespace(rss=rss))
|
||||
monkeypatch.setattr(cpu_worker_mod.psutil, "Process", lambda pid: process)
|
||||
return worker
|
||||
|
||||
|
||||
def test_footprint_above_reservation_asks_for_a_larger_reservation(monkeypatch):
|
||||
# 0.76 GiB already in use against a 0.16 GiB reservation (2% of 8 GiB).
|
||||
worker = _worker(monkeypatch, reserved=int(0.16 * GiB), rss=int(0.76 * GiB))
|
||||
with pytest.raises(ValueError) as exc:
|
||||
worker.determine_available_memory()
|
||||
msg = str(exc.value)
|
||||
assert "increase `--gpu-memory-utilization` to more than 0.10" in msg
|
||||
assert "--kv-cache-memory-bytes" in msg
|
||||
# Neither the misleading remedy nor a negative KV size.
|
||||
assert "other processes" not in msg
|
||||
assert "-0." not in msg
|
||||
|
||||
|
||||
def test_short_node_memory_still_asks_to_free_memory(monkeypatch):
|
||||
worker = _worker(monkeypatch, reserved=6 * GiB, rss=1 * GiB, available=2 * GiB)
|
||||
with pytest.raises(ValueError, match="Reduce CPU memory used by other processes"):
|
||||
worker.determine_available_memory()
|
||||
|
||||
|
||||
def test_budget_that_fits_sizes_the_kv_cache(monkeypatch):
|
||||
worker = _worker(monkeypatch, reserved=3 * GiB, rss=1 * GiB)
|
||||
assert worker.determine_available_memory() == 2 * GiB
|
||||
@@ -234,10 +234,24 @@ class CPUWorker(Worker):
|
||||
else:
|
||||
consumed_memory = psutil.Process(os.getpid()).memory_info().rss
|
||||
requested_memory_for_kv = int(self.requested_cpu_memory - consumed_memory)
|
||||
if (
|
||||
requested_memory_for_kv <= 0
|
||||
or requested_memory_for_kv > available_memory
|
||||
):
|
||||
if requested_memory_for_kv <= 0:
|
||||
# The worker's own footprint already fills its reservation, so
|
||||
# freeing memory elsewhere cannot help: only a larger
|
||||
# reservation or an explicit KV cache size can.
|
||||
in_use = math.ceil(consumed_memory / memory_status.total_memory * 100)
|
||||
raise ValueError(
|
||||
f"The worker already uses {format_gib(consumed_memory)} GiB "
|
||||
"(model weights and runtime), more than the "
|
||||
f"{format_gib(self.requested_cpu_memory)} GiB of CPU memory "
|
||||
f"reserved for it on node {cpu_core.numa_node}, leaving none "
|
||||
"for the KV cache. On the CPU backend, the "
|
||||
"`--gpu-memory-utilization` flag controls the fraction of CPU "
|
||||
"memory reserved (despite its name). To resolve: increase "
|
||||
f"`--gpu-memory-utilization` to more than {in_use / 100:.2f}, "
|
||||
"or set the KV cache size explicitly with "
|
||||
"`--kv-cache-memory-bytes` or VLLM_CPU_KVCACHE_SPACE."
|
||||
)
|
||||
if requested_memory_for_kv > available_memory:
|
||||
raise ValueError(
|
||||
f"Available memory on node {cpu_core.numa_node} "
|
||||
f"({format_gib(available_memory)}/"
|
||||
|
||||
Reference in New Issue
Block a user