[Bugfix][MRV2] Release model and KV cache on in-process engine shutdown (#54162)
Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
3958a420f0
commit
fb68025138
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import weakref
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
@@ -121,7 +122,9 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
|
||||
|
||||
|
||||
# Cache results of `SupportsMultiModal.get_language_model`
|
||||
_language_model_by_module = dict[nn.Module, "VllmModel"]()
|
||||
_language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
Reference in New Issue
Block a user