diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 64f5d8047f..de5b235e20 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -219,6 +219,7 @@ steps: - pytest -v -s -m 'cpu_test' v1/core - pytest -v -s v1/structured_output - pytest -v -s v1/test_serial_utils.py + - pytest -v -s v1/worker/test_cpu_worker_kv_budget.py - pytest -v -s v1/test_kv_cache_spec_registry.py - pytest -v -s v1/cudagraph/test_cudagraph_manager.py - pytest -v -s -m 'cpu_test' v1/kv_connector/unit diff --git a/tests/v1/worker/test_cpu_worker_kv_budget.py b/tests/v1/worker/test_cpu_worker_kv_budget.py new file mode 100644 index 0000000000..e0493d7278 --- /dev/null +++ b/tests/v1/worker/test_cpu_worker_kv_budget.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The CPU worker's automatic KV cache sizing must report the right cause. + +A worker whose own footprint exceeds its reservation and a node that is short +of free memory need opposite remedies, so they get separate errors. +""" + +from types import SimpleNamespace + +import pytest + +from vllm.platforms import current_platform +from vllm.utils.cpu_resource_utils import MemoryNodeInfo + +# Importing the CPU worker patches torch and the GPU buffer utilities for the +# whole process (vllm/v1/worker/cpu/shm.py), so the import is deferred into +# the tests and they run only on the CPU platform. +pytestmark = pytest.mark.skipif( + not current_platform.is_cpu(), reason="CPU worker test; needs the CPU platform" +) + +GiB = 1 << 30 + + +def _worker(monkeypatch, *, reserved, rss, total=8 * GiB, available=4 * GiB): + import vllm.v1.worker.cpu_worker as cpu_worker_mod + from vllm.v1.worker.cpu_worker import CPUWorker + + worker = CPUWorker.__new__(CPUWorker) + worker.requested_cpu_memory = reserved + worker.cache_config = SimpleNamespace(kv_cache_memory_bytes=None) + monkeypatch.setattr(CPUWorker, "_should_warm_up_model", lambda self: False) + monkeypatch.setattr( + cpu_worker_mod, + "get_allowed_cpu_list", + lambda: [SimpleNamespace(numa_node=0)], + ) + monkeypatch.setattr( + cpu_worker_mod, + "get_memory_node_info", + lambda node: MemoryNodeInfo(total_memory=total, available_memory=available), + ) + process = SimpleNamespace(memory_info=lambda: SimpleNamespace(rss=rss)) + monkeypatch.setattr(cpu_worker_mod.psutil, "Process", lambda pid: process) + return worker + + +def test_footprint_above_reservation_asks_for_a_larger_reservation(monkeypatch): + # 0.76 GiB already in use against a 0.16 GiB reservation (2% of 8 GiB). + worker = _worker(monkeypatch, reserved=int(0.16 * GiB), rss=int(0.76 * GiB)) + with pytest.raises(ValueError) as exc: + worker.determine_available_memory() + msg = str(exc.value) + assert "increase `--gpu-memory-utilization` to more than 0.10" in msg + assert "--kv-cache-memory-bytes" in msg + # Neither the misleading remedy nor a negative KV size. + assert "other processes" not in msg + assert "-0." not in msg + + +def test_short_node_memory_still_asks_to_free_memory(monkeypatch): + worker = _worker(monkeypatch, reserved=6 * GiB, rss=1 * GiB, available=2 * GiB) + with pytest.raises(ValueError, match="Reduce CPU memory used by other processes"): + worker.determine_available_memory() + + +def test_budget_that_fits_sizes_the_kv_cache(monkeypatch): + worker = _worker(monkeypatch, reserved=3 * GiB, rss=1 * GiB) + assert worker.determine_available_memory() == 2 * GiB diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py index b103215d65..22b8618f3b 100644 --- a/vllm/v1/worker/cpu_worker.py +++ b/vllm/v1/worker/cpu_worker.py @@ -234,10 +234,24 @@ class CPUWorker(Worker): else: consumed_memory = psutil.Process(os.getpid()).memory_info().rss requested_memory_for_kv = int(self.requested_cpu_memory - consumed_memory) - if ( - requested_memory_for_kv <= 0 - or requested_memory_for_kv > available_memory - ): + if requested_memory_for_kv <= 0: + # The worker's own footprint already fills its reservation, so + # freeing memory elsewhere cannot help: only a larger + # reservation or an explicit KV cache size can. + in_use = math.ceil(consumed_memory / memory_status.total_memory * 100) + raise ValueError( + f"The worker already uses {format_gib(consumed_memory)} GiB " + "(model weights and runtime), more than the " + f"{format_gib(self.requested_cpu_memory)} GiB of CPU memory " + f"reserved for it on node {cpu_core.numa_node}, leaving none " + "for the KV cache. On the CPU backend, the " + "`--gpu-memory-utilization` flag controls the fraction of CPU " + "memory reserved (despite its name). To resolve: increase " + f"`--gpu-memory-utilization` to more than {in_use / 100:.2f}, " + "or set the KV cache size explicitly with " + "`--kv-cache-memory-bytes` or VLLM_CPU_KVCACHE_SPACE." + ) + if requested_memory_for_kv > available_memory: raise ValueError( f"Available memory on node {cpu_core.numa_node} " f"({format_gib(available_memory)}/"