From 1c47ebfc57d9e2415486e6080027ba319c73362a Mon Sep 17 00:00:00 2001 From: Praneeth Reddy A <72988597+0xViking@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:13:55 +0530 Subject: [PATCH] [Docs] Fix VLLM_CPU_KVCACHE_SPACE default and units (#60053) Signed-off-by: Praneeth Co-authored-by: Li, Jiang --- docs/configuration/conserving_memory.md | 2 +- docs/getting_started/installation/cpu.md | 2 +- vllm/platforms/cpu.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration/conserving_memory.md b/docs/configuration/conserving_memory.md index 96a903bc31..b397283a62 100644 --- a/docs/configuration/conserving_memory.md +++ b/docs/configuration/conserving_memory.md @@ -80,7 +80,7 @@ llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct", enforce_eager=True) If you run out of CPU RAM, try the following options: - (Multi-modal models only) you can set the size of multi-modal cache by setting `mm_processor_cache_gb` engine argument (default 4 GiB). -- (CPU backend only) you can set the size of KV cache using `VLLM_CPU_KVCACHE_SPACE` environment variable (default 4 GiB). +- (CPU backend only) you can set the size of KV cache using `VLLM_CPU_KVCACHE_SPACE` environment variable, in GiB (default 0, meaning vLLM doesn't override the KV cache size through this variable). ## Multi-modal input limits diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index d277534138..1e58a68fcf 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -293,7 +293,7 @@ See [AMD Zen optimizations](#amd-zen-optimizations) for detection rules, support ### How to decide `VLLM_CPU_KVCACHE_SPACE`? -This value is 4GB by default. Larger space can support more concurrent requests, longer context length. However, users should take care of memory capacity of each NUMA node. The memory usage of each TP rank is the sum of `weight shard size` and `VLLM_CPU_KVCACHE_SPACE`, if it exceeds the capacity of a single NUMA node, the TP worker will be killed with `exitcode 9` due to out-of-memory. +This value is 0 by default, which means vLLM doesn't override the KV cache size through this variable. When set, the value is in GiB (e.g, `VLLM_CPU_KVCACHE_SPACE`=40 means 40 GiB). Larger space can support more concurrent requests, longer context length. However, users should take care of memory capacity of each NUMA node. The memory usage of each TP rank is the sum of `weight shard size` and `VLLM_CPU_KVCACHE_SPACE`, if it exceeds the capacity of a single NUMA node, the TP worker will be killed with `exitcode 9` due to out-of-memory. This is a legacy setting; use `--kv-cache-memory-bytes` instead. ### How to do performance tuning for vLLM CPU? diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index e0257acb80..43d472f22e 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -323,7 +323,7 @@ class CpuPlatform(Platform): mamba_backend, ) - # Lagecy setting + # Legacy setting env_key = "VLLM_CPU_KVCACHE_SPACE" if env_key in os.environ and os.environ[env_key] != "": kv_cache_space = int(os.environ[env_key])