[Bugfix][V1] Clean up compiled-model bytecode hooks on VllmRunner exit (#45195)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ting SUN
2026-06-16 20:31:17 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 14b438a98b
commit 7b5d60cc37
4 changed files with 61 additions and 2 deletions
+23
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
import weakref
from collections.abc import Callable, Mapping
from copy import copy
from typing import Any
@@ -123,6 +124,14 @@ class LLMEngine:
# for v0 compatibility
self.model_executor = self.engine_core.engine_core.model_executor # type: ignore
# Capture the model while reachable so the finalizer can drop the
# bytecode hooks pinning it (frees GPU memory on engine deletion).
model = self._get_driver_model_for_cleanup()
if model is not None:
self._finalizer = weakref.finalize(
self, LLMEngine._cleanup_instance_caches, model
)
if self.external_launcher_dp:
# If we use DP in external launcher mode, we reuse the
# existing DP group used for data communication.
@@ -419,6 +428,20 @@ class LLMEngine:
def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
return self.collective_rpc("apply_model", args=(func,))
def _get_driver_model_for_cleanup(self) -> nn.Module | None:
driver_worker = getattr(self.model_executor, "driver_worker", None)
model_runner = getattr(driver_worker, "model_runner", None)
return getattr(model_runner, "model", None)
@staticmethod
def _cleanup_instance_caches(model) -> None:
"""Remove the bytecode hooks that pin the compiled model."""
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
for module in model.modules():
if isinstance(module, TorchCompileWithNoGuardsWrapper):
module.cleanup()
def __del__(self):
dp_group = getattr(self, "dp_group", None)
if dp_group is not None and not self.external_launcher_dp: