[Feature] Offload KV-init runtime state on sleep (#59158)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-02 09:49:21 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c15672cfb4
commit 8e58ac22af
4 changed files with 45 additions and 4 deletions
+10 -3
View File
@@ -841,10 +841,17 @@ class Worker(WorkerBase):
if mem_pool_context is None:
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
self.model_runner.initialize_kv_cache(
kv_cache_config,
kv_cache_allocation_context=mem_pool_context,
# Offload KV-init state, except on XPU, whose outermost pool would win.
runtime_pool = (
nullcontext()
if current_platform.is_xpu()
else self._maybe_get_memory_pool_context(tag="runtime")
)
with runtime_pool:
self.model_runner.initialize_kv_cache(
kv_cache_config,
kv_cache_allocation_context=mem_pool_context,
)
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
# GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch