[Bugfix][MRV2] Release model and KV cache on in-process engine shutdown (#54162)

Signed-off-by: Oxana Korzh <okorzh@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oxana Korzh
2026-08-28 22:09:08 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 3958a420f0
commit fb68025138
4 changed files with 142 additions and 3 deletions
+4 -1
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import weakref
from collections.abc import (
AsyncGenerator,
Callable,
@@ -121,7 +122,9 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
# Cache results of `SupportsMultiModal.get_language_model`
_language_model_by_module = dict[nn.Module, "VllmModel"]()
_language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = (
weakref.WeakKeyDictionary()
)
@runtime_checkable