[Feature] Offload KV-init runtime state on sleep (#59158)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c15672cfb4
commit
8e58ac22af
@@ -6531,6 +6531,9 @@ class GPUModelRunner(
|
||||
if hasattr(self, "kv_cache_config"):
|
||||
delattr(self, "kv_cache_config")
|
||||
self.cache_config.num_gpu_blocks = None
|
||||
# Profiling may have rebuilt the InputBatch outside the worker's
|
||||
# "runtime" pool; the real initialize_kv_cache rebuilds it.
|
||||
self._init_block_sizes = []
|
||||
|
||||
for layer in self.compilation_config.static_forward_context.values():
|
||||
if hasattr(layer, "kv_cache"):
|
||||
|
||||
Reference in New Issue
Block a user