[Bugfix][MRV2] Release model and KV cache on in-process engine shutdown (#54162)

Signed-off-by: Oxana Korzh <okorzh@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oxana Korzh
2026-08-28 22:09:08 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 3958a420f0
commit fb68025138
4 changed files with 142 additions and 3 deletions
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The `get_language_model` cache must not own the models it keys on.
A strong-keyed module-level dict pins every model ever loaded in the process
for the life of the interpreter. In-process teardown
(``VLLM_ENABLE_V1_MULTIPROCESSING=0``) then keeps the weights -- and the KV
cache aliased onto the model's layers -- resident after engine shutdown.
"""
import gc
import weakref
import pytest
import torch.nn as nn
from vllm.model_executor.models.interfaces import _language_model_by_module
pytestmark = pytest.mark.cpu_test
class _LanguageModel(nn.Module):
def embed_input_ids(self, input_ids): # pragma: no cover - never called
raise NotImplementedError
class _MultiModalModel(nn.Module):
def __init__(self):
super().__init__()
self.language_model = _LanguageModel()
def test_cache_is_weak_keyed():
assert isinstance(_language_model_by_module, weakref.WeakKeyDictionary)
def test_cached_entry_does_not_pin_the_model():
model = _MultiModalModel()
_language_model_by_module[model] = model.language_model
ref = weakref.ref(model)
del model
gc.collect()
assert ref() is None, (
"the get_language_model cache is keeping the model alive; in-process "
"engine shutdown will not release its weights or KV cache"
)
def test_cached_entry_is_dropped_with_its_key():
model = _MultiModalModel()
_language_model_by_module[model] = model.language_model
assert model in _language_model_by_module
del model
gc.collect()
assert not any(isinstance(k, _MultiModalModel) for k in _language_model_by_module)
@@ -0,0 +1,74 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""`LLMEngine`'s cleanup finalizer must not own the model.
``weakref.finalize`` keeps its ``args`` strongly in a class-level registry
until it fires, and it only fires once the ``LLMEngine`` itself is collected.
Handing it the model directly therefore pins the weights -- and the KV cache
aliased onto the model's layers -- for as long as the engine is reachable,
which in-process is past engine shutdown.
"""
import gc
import weakref
import pytest
import torch.nn as nn
from vllm.v1.engine.llm_engine import LLMEngine
pytestmark = pytest.mark.cpu_test
class _Model(nn.Module):
pass
def test_finalize_args_are_held_strongly():
"""The premise: this is why the model must be passed weakly."""
class Owner:
pass
owner, model = Owner(), _Model()
weakref.finalize(owner, lambda _m: None, model)
ref = weakref.ref(model)
del model
gc.collect()
assert ref() is not None
del owner
gc.collect()
assert ref() is None
def test_cleanup_tolerates_a_dead_model():
model = _Model()
ref = weakref.ref(model)
del model
gc.collect()
assert ref() is None
LLMEngine._cleanup_instance_caches(ref)
def test_cleanup_still_walks_a_live_model(monkeypatch):
from vllm.compilation import wrapper
cleaned = []
class _FakeWrapper(nn.Module):
def cleanup(self):
cleaned.append(self)
monkeypatch.setattr(
wrapper, "TorchCompileWithNoGuardsWrapper", _FakeWrapper, raising=False
)
model = _Model()
model.inner = _FakeWrapper()
LLMEngine._cleanup_instance_caches(weakref.ref(model))
assert cleaned == [model.inner]
+4 -1
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import weakref
from collections.abc import (
AsyncGenerator,
Callable,
@@ -121,7 +122,9 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
# Cache results of `SupportsMultiModal.get_language_model`
_language_model_by_module = dict[nn.Module, "VllmModel"]()
_language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = (
weakref.WeakKeyDictionary()
)
@runtime_checkable
+5 -2
View File
@@ -129,7 +129,7 @@ class LLMEngine:
model = self._get_driver_model_for_cleanup()
if model is not None:
self._finalizer = weakref.finalize(
self, LLMEngine._cleanup_instance_caches, model
self, LLMEngine._cleanup_instance_caches, weakref.ref(model)
)
if self.external_launcher_dp:
@@ -444,10 +444,13 @@ class LLMEngine:
return getattr(model_runner, "model", None)
@staticmethod
def _cleanup_instance_caches(model) -> None:
def _cleanup_instance_caches(model_ref: "weakref.ref[nn.Module]") -> None:
"""Remove the bytecode hooks that pin the compiled model."""
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
model = model_ref()
if model is None:
return
for module in model.modules():
if isinstance(module, TorchCompileWithNoGuardsWrapper):
module.cleanup()