[UX][Startup] Account for CUDA graphs during memory profiling (#30515)

This commit is contained in:
Matthew Bonanni
2026-03-07 13:49:23 -08:00
committed by GitHub
parent 85f50eb41f
commit ebb9cc5f2b
6 changed files with 360 additions and 61 deletions
+19 -1
View File
@@ -2,10 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import weakref
from collections import Counter
from collections.abc import Callable
from contextlib import ExitStack
from typing import Any
from typing import Any, ClassVar
from unittest.mock import patch
import torch
@@ -162,6 +163,14 @@ class CUDAGraphWrapper:
guaranteed when VLLM_LOGGING_LEVEL == "DEBUG".
"""
_all_instances: ClassVar[weakref.WeakSet["CUDAGraphWrapper"]] = weakref.WeakSet()
@classmethod
def clear_all_graphs(cls) -> None:
"""Clear captured graphs from all CUDAGraphWrapper instances."""
for instance in list(cls._all_instances):
instance.clear_graphs()
def __init__(
self,
runnable: Callable[..., Any],
@@ -192,6 +201,8 @@ class CUDAGraphWrapper:
# cudagraphs for.
self.concrete_cudagraph_entries: dict[BatchDescriptor, CUDAGraphEntry] = {}
CUDAGraphWrapper._all_instances.add(self)
def __getattr__(self, key: str) -> Any:
# allow accessing the attributes of the runnable.
if hasattr(self.runnable, key):
@@ -205,6 +216,13 @@ class CUDAGraphWrapper:
# in case we need to access the original runnable.
return self.runnable
@property
def cudagraph_wrapper(self) -> "CUDAGraphWrapper":
return self
def clear_graphs(self) -> None:
self.concrete_cudagraph_entries.clear()
def __call__(self, *args: Any, **kwargs: Any) -> Any | None:
forward_context = get_forward_context()
batch_descriptor = forward_context.batch_descriptor