[Bugfix][KV Offload] Allow explicit SimpleCPUOffload block counts (#61674)
Signed-off-by: Sherif Waly <sherif.waly@mistral.ai> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
vllm-agent
parent
71c20f73f2
commit
ce8853d27a
@@ -101,6 +101,16 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
cpu_capacity_per_rank = explicit
|
||||
|
||||
# When set, num_cpu_blocks overrides automatic pool sizing from byte budgets
|
||||
# (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks
|
||||
# with different KV layouts (e.g. PP stages) to use the same pool size.
|
||||
# Byte budgets still bound capacity.
|
||||
num_cpu_blocks = extra_config.get("num_cpu_blocks")
|
||||
if num_cpu_blocks is not None and (
|
||||
type(num_cpu_blocks) is not int or num_cpu_blocks <= 0
|
||||
):
|
||||
raise ValueError("num_cpu_blocks must be a positive integer")
|
||||
|
||||
lazy_offload = bool(extra_config.get("lazy_offload", False))
|
||||
|
||||
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
|
||||
@@ -160,6 +170,12 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
kv_offload_backend,
|
||||
disk_path or "none",
|
||||
)
|
||||
if num_cpu_blocks is not None:
|
||||
logger.info(
|
||||
"Using explicit num_cpu_blocks=%d for offload pool sizing "
|
||||
"instead of automatic sizing from byte budgets.",
|
||||
num_cpu_blocks,
|
||||
)
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
@@ -177,6 +193,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
@@ -188,6 +205,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
disk_capacity_bytes=disk_capacity_bytes,
|
||||
disk_buffer_slots=disk_buffer_slots,
|
||||
use_page_cache=use_page_cache,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
|
||||
# --- Worker-side methods ---
|
||||
|
||||
Reference in New Issue
Block a user