diff --git a/tests/models/test_language_model_cache_is_weak.py b/tests/models/test_language_model_cache_is_weak.py new file mode 100644 index 0000000000..19d93e479e --- /dev/null +++ b/tests/models/test_language_model_cache_is_weak.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The `get_language_model` cache must not own the models it keys on. + +A strong-keyed module-level dict pins every model ever loaded in the process +for the life of the interpreter. In-process teardown +(``VLLM_ENABLE_V1_MULTIPROCESSING=0``) then keeps the weights -- and the KV +cache aliased onto the model's layers -- resident after engine shutdown. +""" + +import gc +import weakref + +import pytest +import torch.nn as nn + +from vllm.model_executor.models.interfaces import _language_model_by_module + +pytestmark = pytest.mark.cpu_test + + +class _LanguageModel(nn.Module): + def embed_input_ids(self, input_ids): # pragma: no cover - never called + raise NotImplementedError + + +class _MultiModalModel(nn.Module): + def __init__(self): + super().__init__() + self.language_model = _LanguageModel() + + +def test_cache_is_weak_keyed(): + assert isinstance(_language_model_by_module, weakref.WeakKeyDictionary) + + +def test_cached_entry_does_not_pin_the_model(): + model = _MultiModalModel() + _language_model_by_module[model] = model.language_model + ref = weakref.ref(model) + + del model + gc.collect() + + assert ref() is None, ( + "the get_language_model cache is keeping the model alive; in-process " + "engine shutdown will not release its weights or KV cache" + ) + + +def test_cached_entry_is_dropped_with_its_key(): + model = _MultiModalModel() + _language_model_by_module[model] = model.language_model + assert model in _language_model_by_module + + del model + gc.collect() + + assert not any(isinstance(k, _MultiModalModel) for k in _language_model_by_module) diff --git a/tests/v1/engine/test_llm_engine_finalizer_is_weak.py b/tests/v1/engine/test_llm_engine_finalizer_is_weak.py new file mode 100644 index 0000000000..ee05973e75 --- /dev/null +++ b/tests/v1/engine/test_llm_engine_finalizer_is_weak.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""`LLMEngine`'s cleanup finalizer must not own the model. + +``weakref.finalize`` keeps its ``args`` strongly in a class-level registry +until it fires, and it only fires once the ``LLMEngine`` itself is collected. +Handing it the model directly therefore pins the weights -- and the KV cache +aliased onto the model's layers -- for as long as the engine is reachable, +which in-process is past engine shutdown. +""" + +import gc +import weakref + +import pytest +import torch.nn as nn + +from vllm.v1.engine.llm_engine import LLMEngine + +pytestmark = pytest.mark.cpu_test + + +class _Model(nn.Module): + pass + + +def test_finalize_args_are_held_strongly(): + """The premise: this is why the model must be passed weakly.""" + + class Owner: + pass + + owner, model = Owner(), _Model() + weakref.finalize(owner, lambda _m: None, model) + ref = weakref.ref(model) + + del model + gc.collect() + + assert ref() is not None + del owner + gc.collect() + assert ref() is None + + +def test_cleanup_tolerates_a_dead_model(): + model = _Model() + ref = weakref.ref(model) + del model + gc.collect() + assert ref() is None + + LLMEngine._cleanup_instance_caches(ref) + + +def test_cleanup_still_walks_a_live_model(monkeypatch): + from vllm.compilation import wrapper + + cleaned = [] + + class _FakeWrapper(nn.Module): + def cleanup(self): + cleaned.append(self) + + monkeypatch.setattr( + wrapper, "TorchCompileWithNoGuardsWrapper", _FakeWrapper, raising=False + ) + + model = _Model() + model.inner = _FakeWrapper() + + LLMEngine._cleanup_instance_caches(weakref.ref(model)) + + assert cleaned == [model.inner] diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index fd81bb502f..f35dc3b9cc 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import weakref from collections.abc import ( AsyncGenerator, Callable, @@ -121,7 +122,9 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor: # Cache results of `SupportsMultiModal.get_language_model` -_language_model_by_module = dict[nn.Module, "VllmModel"]() +_language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = ( + weakref.WeakKeyDictionary() +) @runtime_checkable diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 0db83b95f2..38a90a4b67 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -129,7 +129,7 @@ class LLMEngine: model = self._get_driver_model_for_cleanup() if model is not None: self._finalizer = weakref.finalize( - self, LLMEngine._cleanup_instance_caches, model + self, LLMEngine._cleanup_instance_caches, weakref.ref(model) ) if self.external_launcher_dp: @@ -444,10 +444,13 @@ class LLMEngine: return getattr(model_runner, "model", None) @staticmethod - def _cleanup_instance_caches(model) -> None: + def _cleanup_instance_caches(model_ref: "weakref.ref[nn.Module]") -> None: """Remove the bytecode hooks that pin the compiled model.""" from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper + model = model_ref() + if model is None: + return for module in model.modules(): if isinstance(module, TorchCompileWithNoGuardsWrapper): module.cleanup()