[Feature] Offload KV-init runtime state on sleep (#59158)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c15672cfb4
commit
8e58ac22af
@@ -841,10 +841,17 @@ class Worker(WorkerBase):
|
||||
if mem_pool_context is None:
|
||||
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
|
||||
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=mem_pool_context,
|
||||
# Offload KV-init state, except on XPU, whose outermost pool would win.
|
||||
runtime_pool = (
|
||||
nullcontext()
|
||||
if current_platform.is_xpu()
|
||||
else self._maybe_get_memory_pool_context(tag="runtime")
|
||||
)
|
||||
with runtime_pool:
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=mem_pool_context,
|
||||
)
|
||||
|
||||
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
|
||||
# GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch
|
||||
|
||||
Reference in New Issue
Block a user