[Feature] Offload KV-init runtime state on sleep (#59158)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-02 09:49:21 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c15672cfb4
commit 8e58ac22af
4 changed files with 45 additions and 4 deletions
+3
View File
@@ -6531,6 +6531,9 @@ class GPUModelRunner(
if hasattr(self, "kv_cache_config"):
delattr(self, "kv_cache_config")
self.cache_config.num_gpu_blocks = None
# Profiling may have rebuilt the InputBatch outside the worker's
# "runtime" pool; the real initialize_kv_cache rebuilds it.
self._init_block_sizes = []
for layer in self.compilation_config.static_forward_context.values():
if hasattr(layer, "kv_cache"):