[Bugfix][MRV2] Release model and KV cache on in-process engine shutdown (#54162)
Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
3958a420f0
commit
fb68025138
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The `get_language_model` cache must not own the models it keys on.
|
||||
|
||||
A strong-keyed module-level dict pins every model ever loaded in the process
|
||||
for the life of the interpreter. In-process teardown
|
||||
(``VLLM_ENABLE_V1_MULTIPROCESSING=0``) then keeps the weights -- and the KV
|
||||
cache aliased onto the model's layers -- resident after engine shutdown.
|
||||
"""
|
||||
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.models.interfaces import _language_model_by_module
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _LanguageModel(nn.Module):
|
||||
def embed_input_ids(self, input_ids): # pragma: no cover - never called
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class _MultiModalModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.language_model = _LanguageModel()
|
||||
|
||||
|
||||
def test_cache_is_weak_keyed():
|
||||
assert isinstance(_language_model_by_module, weakref.WeakKeyDictionary)
|
||||
|
||||
|
||||
def test_cached_entry_does_not_pin_the_model():
|
||||
model = _MultiModalModel()
|
||||
_language_model_by_module[model] = model.language_model
|
||||
ref = weakref.ref(model)
|
||||
|
||||
del model
|
||||
gc.collect()
|
||||
|
||||
assert ref() is None, (
|
||||
"the get_language_model cache is keeping the model alive; in-process "
|
||||
"engine shutdown will not release its weights or KV cache"
|
||||
)
|
||||
|
||||
|
||||
def test_cached_entry_is_dropped_with_its_key():
|
||||
model = _MultiModalModel()
|
||||
_language_model_by_module[model] = model.language_model
|
||||
assert model in _language_model_by_module
|
||||
|
||||
del model
|
||||
gc.collect()
|
||||
|
||||
assert not any(isinstance(k, _MultiModalModel) for k in _language_model_by_module)
|
||||
@@ -0,0 +1,74 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""`LLMEngine`'s cleanup finalizer must not own the model.
|
||||
|
||||
``weakref.finalize`` keeps its ``args`` strongly in a class-level registry
|
||||
until it fires, and it only fires once the ``LLMEngine`` itself is collected.
|
||||
Handing it the model directly therefore pins the weights -- and the KV cache
|
||||
aliased onto the model's layers -- for as long as the engine is reachable,
|
||||
which in-process is past engine shutdown.
|
||||
"""
|
||||
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _Model(nn.Module):
|
||||
pass
|
||||
|
||||
|
||||
def test_finalize_args_are_held_strongly():
|
||||
"""The premise: this is why the model must be passed weakly."""
|
||||
|
||||
class Owner:
|
||||
pass
|
||||
|
||||
owner, model = Owner(), _Model()
|
||||
weakref.finalize(owner, lambda _m: None, model)
|
||||
ref = weakref.ref(model)
|
||||
|
||||
del model
|
||||
gc.collect()
|
||||
|
||||
assert ref() is not None
|
||||
del owner
|
||||
gc.collect()
|
||||
assert ref() is None
|
||||
|
||||
|
||||
def test_cleanup_tolerates_a_dead_model():
|
||||
model = _Model()
|
||||
ref = weakref.ref(model)
|
||||
del model
|
||||
gc.collect()
|
||||
assert ref() is None
|
||||
|
||||
LLMEngine._cleanup_instance_caches(ref)
|
||||
|
||||
|
||||
def test_cleanup_still_walks_a_live_model(monkeypatch):
|
||||
from vllm.compilation import wrapper
|
||||
|
||||
cleaned = []
|
||||
|
||||
class _FakeWrapper(nn.Module):
|
||||
def cleanup(self):
|
||||
cleaned.append(self)
|
||||
|
||||
monkeypatch.setattr(
|
||||
wrapper, "TorchCompileWithNoGuardsWrapper", _FakeWrapper, raising=False
|
||||
)
|
||||
|
||||
model = _Model()
|
||||
model.inner = _FakeWrapper()
|
||||
|
||||
LLMEngine._cleanup_instance_caches(weakref.ref(model))
|
||||
|
||||
assert cleaned == [model.inner]
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import weakref
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
@@ -121,7 +122,9 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
|
||||
|
||||
|
||||
# Cache results of `SupportsMultiModal.get_language_model`
|
||||
_language_model_by_module = dict[nn.Module, "VllmModel"]()
|
||||
_language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
@@ -129,7 +129,7 @@ class LLMEngine:
|
||||
model = self._get_driver_model_for_cleanup()
|
||||
if model is not None:
|
||||
self._finalizer = weakref.finalize(
|
||||
self, LLMEngine._cleanup_instance_caches, model
|
||||
self, LLMEngine._cleanup_instance_caches, weakref.ref(model)
|
||||
)
|
||||
|
||||
if self.external_launcher_dp:
|
||||
@@ -444,10 +444,13 @@ class LLMEngine:
|
||||
return getattr(model_runner, "model", None)
|
||||
|
||||
@staticmethod
|
||||
def _cleanup_instance_caches(model) -> None:
|
||||
def _cleanup_instance_caches(model_ref: "weakref.ref[nn.Module]") -> None:
|
||||
"""Remove the bytecode hooks that pin the compiled model."""
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
|
||||
model = model_ref()
|
||||
if model is None:
|
||||
return
|
||||
for module in model.modules():
|
||||
if isinstance(module, TorchCompileWithNoGuardsWrapper):
|
||||
module.cleanup()
|
||||
|
||||
Reference in New Issue
Block a user