[Feat][Core] Add disk offloading support to SimpleCPUOffloadConnector (#49644)

Signed-off-by: Guanyi Chen <939416532@qq.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Guanyi Chen
2026-08-07 15:37:47 +08:00
committed by GitHub
co-authored by Claude
parent 21ea5b4fa1
commit 58fcaa0baa
4 changed files with 466 additions and 19 deletions
@@ -41,6 +41,15 @@ logger = init_logger(__name__)
# Default CPU capacity: 8 GB
DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
VALID_KV_OFFLOAD_BACKENDS = ("cpu", "disk")
# Keys that only apply to the disk backend, warned about under "cpu".
_DISK_ONLY_KEYS = (
"disk_path",
"disk_capacity_bytes",
"disk_buffer_slots",
"use_page_cache",
)
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
@@ -82,6 +91,36 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
lazy_offload = bool(extra_config.get("lazy_offload", False))
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
if kv_offload_backend not in VALID_KV_OFFLOAD_BACKENDS:
raise ValueError(
f"Unknown kv_offload_backend {kv_offload_backend!r}; "
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
)
disk_mode = kv_offload_backend == "disk"
disk_path = extra_config.get("disk_path", None) or None
disk_capacity_bytes = int(
extra_config.get("disk_capacity_bytes", 100 * (1024**3))
)
disk_buffer_slots = max(1, int(extra_config.get("disk_buffer_slots", 2)))
use_page_cache = bool(extra_config.get("use_page_cache", False))
if disk_mode:
if disk_path is None:
raise ValueError(
'kv_offload_backend="disk" requires disk_path to be set.'
)
else:
ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config]
if ignored:
logger.warning(
'kv_offload_backend is "cpu", ignoring disk-only config: %s. '
'Set kv_offload_backend="disk" to enable the disk backend.',
", ".join(ignored),
)
disk_path = None
self.scheduler_manager: SimpleCPUOffloadScheduler | None = None
self.worker_handler: SimpleCPUOffloadWorker | None = None
@@ -94,11 +133,13 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
logger.info(
"SimpleCPUOffloadConnector: role=%s, "
"per_rank=%.2f GB, world_size=%d, mode=%s",
"per_rank=%.2f GB, world_size=%d, mode=%s, backend=%s, disk=%s",
role.name,
cpu_capacity_per_rank / (1024**3),
world_size,
"lazy" if lazy_offload else "eager",
kv_offload_backend,
disk_path or "none",
)
if role == KVConnectorRole.SCHEDULER:
@@ -115,10 +156,18 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
vllm_config, kv_cache_config, cpu_capacity_per_rank
vllm_config,
kv_cache_config,
cpu_capacity_per_rank,
kv_offload_backend=kv_offload_backend,
disk_path=disk_path,
disk_capacity_bytes=disk_capacity_bytes,
disk_buffer_slots=disk_buffer_slots,
use_page_cache=use_page_cache,
)
# --- Worker-side methods ---