[V0 Deprecation] Refactor kv cache from list to element (#37487)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-03-23 20:10:11 -07:00
committed by GitHub
parent de99d91ece
commit c59a132f96
27 changed files with 70 additions and 85 deletions
+4 -1
View File
@@ -5830,7 +5830,10 @@ class GPUModelRunner(
for layer in self.compilation_config.static_forward_context.values():
if hasattr(layer, "kv_cache"):
layer.kv_cache = []
kv_cache = layer.kv_cache
layer.kv_cache = (
torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else []
)
gc.collect()
torch.accelerator.empty_cache()