[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
+75
-21
@@ -3,11 +3,12 @@
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Hashable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from itertools import accumulate
|
||||
from math import prod
|
||||
from typing import TypeVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -17,6 +18,7 @@ from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def _compute_bytes(shape: tuple[int, ...], dtype: torch.dtype) -> int:
|
||||
@@ -47,8 +49,8 @@ def use_workspace_lane(lane: int) -> Iterator[None]:
|
||||
class WorkspaceManager:
|
||||
"""Manager for workspace allocation.
|
||||
|
||||
Manages one workspace buffer per active ``(ubatch, lane)`` slot.
|
||||
Can be locked to prevent further growth during execution.
|
||||
Manages shared scratch and keyed persistent resources per ``(ubatch, lane)``.
|
||||
Can be locked to prevent growth or new resources during execution.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -66,6 +68,9 @@ class WorkspaceManager:
|
||||
self._current_workspaces: list[torch.Tensor | None] = [None] * (
|
||||
self._num_ubatches * self._num_lanes
|
||||
)
|
||||
self._persistent_resources: list[dict[Hashable, object]] = [
|
||||
{} for _ in self._current_workspaces
|
||||
]
|
||||
self._locked: bool = False
|
||||
|
||||
@staticmethod
|
||||
@@ -76,10 +81,10 @@ class WorkspaceManager:
|
||||
return workspace.numel() * workspace.element_size()
|
||||
|
||||
def lock(self) -> None:
|
||||
"""Lock the workspace to prevent further growth.
|
||||
"""Lock the workspace to prevent growth or new persistent resources.
|
||||
|
||||
After locking, any attempt to allocate a larger workspace will raise
|
||||
an assertion error. This ensures workspace size is fixed during execution.
|
||||
Larger scratch requests and new persistent resources raise an assertion
|
||||
error. Existing resources retain their storage throughout execution.
|
||||
"""
|
||||
self._locked = True
|
||||
if envs.VLLM_DEBUG_WORKSPACE:
|
||||
@@ -113,6 +118,63 @@ class WorkspaceManager:
|
||||
"""Check if workspace is locked."""
|
||||
return self._locked
|
||||
|
||||
def _get_workspace_id(self) -> int:
|
||||
lane = _workspace_lane.get()
|
||||
if lane >= self._num_lanes:
|
||||
raise RuntimeError(
|
||||
f"Workspace lane {lane} is not configured; manager has "
|
||||
f"{self._num_lanes} lane(s)."
|
||||
)
|
||||
return dbo_current_ubatch_id() * self._num_lanes + lane
|
||||
|
||||
def get_persistent_resource(self, key: Hashable, factory: Callable[[], T]) -> T:
|
||||
"""Return the resource cached for ``key`` in the current ubatch and lane.
|
||||
|
||||
The first request calls ``factory`` and stores its result for the lifetime
|
||||
of this manager. Later requests return the same object without calling
|
||||
the factory again. Each ubatch/lane pair has a separate cache. Once the
|
||||
manager is locked, requesting a key missing from that cache raises an
|
||||
AssertionError.
|
||||
|
||||
Keys must identify the resource type and configuration. Callers must not
|
||||
resize or replace tensor storage held by a cached resource. Uses of the
|
||||
same resource must not overlap; include the CUDA stream in the key when
|
||||
streams can execute concurrently.
|
||||
|
||||
With torch.compile, call this inside a runtime custom op so resource
|
||||
selection happens during execution.
|
||||
"""
|
||||
resources = self._persistent_resources[self._get_workspace_id()]
|
||||
if key not in resources:
|
||||
if self._locked:
|
||||
raise AssertionError(
|
||||
f"Workspace is locked but {key!r} was not allocated during "
|
||||
"warmup for the current ubatch and lane."
|
||||
)
|
||||
resources[key] = factory()
|
||||
return cast(T, resources[key])
|
||||
|
||||
def get_persistent(
|
||||
self,
|
||||
key: Hashable,
|
||||
shape: tuple[int, ...],
|
||||
dtype: torch.dtype,
|
||||
*,
|
||||
zero_init: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Get a fixed-shape tensor, optionally zeroed on first allocation only."""
|
||||
factory = torch.zeros if zero_init else torch.empty
|
||||
workspace = self.get_persistent_resource(
|
||||
(torch.Tensor, key),
|
||||
lambda: factory(shape, dtype=dtype, device=self._device),
|
||||
)
|
||||
if workspace.shape != shape or workspace.dtype != dtype:
|
||||
raise ValueError(
|
||||
f"Persistent workspace {key!r} has shape={workspace.shape}, "
|
||||
f"dtype={workspace.dtype}; requested shape={shape}, dtype={dtype}."
|
||||
)
|
||||
return workspace
|
||||
|
||||
def get_simultaneous(
|
||||
self, *shapes_and_dtypes: tuple[tuple[int, ...], torch.dtype]
|
||||
) -> list[torch.Tensor]:
|
||||
@@ -151,14 +213,7 @@ class WorkspaceManager:
|
||||
The current workspace tensor.
|
||||
|
||||
"""
|
||||
ubatch_id = dbo_current_ubatch_id()
|
||||
lane = _workspace_lane.get()
|
||||
if lane >= self._num_lanes:
|
||||
raise RuntimeError(
|
||||
f"Workspace lane {lane} is not configured; manager has "
|
||||
f"{self._num_lanes} lane(s)."
|
||||
)
|
||||
workspace_id = ubatch_id * self._num_lanes + lane
|
||||
workspace_id = self._get_workspace_id()
|
||||
current_workspace = self._current_workspaces[workspace_id]
|
||||
current_size = self._workspace_size_bytes(current_workspace)
|
||||
|
||||
@@ -218,8 +273,8 @@ class WorkspaceManager:
|
||||
get_caller_info(),
|
||||
current_size / _MB,
|
||||
required_bytes / _MB,
|
||||
ubatch_id,
|
||||
lane,
|
||||
workspace_id // self._num_lanes,
|
||||
workspace_id % self._num_lanes,
|
||||
)
|
||||
|
||||
return current_workspace
|
||||
@@ -277,12 +332,11 @@ def init_workspace_manager(
|
||||
|
||||
|
||||
def lock_workspace() -> None:
|
||||
"""Lock the workspace to prevent further growth.
|
||||
"""Lock the workspace to prevent growth or new persistent resources.
|
||||
|
||||
After calling this function, any attempt to allocate a workspace larger
|
||||
than the current size will raise an AssertionError. This ensures that
|
||||
workspace size is fixed during execution and prevents unexpected memory
|
||||
allocations in the hot path.
|
||||
Larger scratch requests and new persistent resources raise an AssertionError.
|
||||
All required resources must be initialized in each execution slot during
|
||||
warmup, so their storage remains fixed during execution.
|
||||
|
||||
Example:
|
||||
# During initialization
|
||||
|
||||
Reference in New Issue
Block a user