[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
Misha Goin
2026-09-20 17:28:33 +08:00
committed by GitHub
co-authored by ErinYin
parent 0e110f696d
commit e5fce7b56b
21 changed files with 598 additions and 211 deletions
+75 -21
View File
@@ -3,11 +3,12 @@
import inspect
import os
from collections.abc import Iterator
from collections.abc import Callable, Hashable, Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from itertools import accumulate
from math import prod
from typing import TypeVar, cast
import torch
@@ -17,6 +18,7 @@ from vllm.utils.math_utils import round_up
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
logger = init_logger(__name__)
T = TypeVar("T")
def _compute_bytes(shape: tuple[int, ...], dtype: torch.dtype) -> int:
@@ -47,8 +49,8 @@ def use_workspace_lane(lane: int) -> Iterator[None]:
class WorkspaceManager:
"""Manager for workspace allocation.
Manages one workspace buffer per active ``(ubatch, lane)`` slot.
Can be locked to prevent further growth during execution.
Manages shared scratch and keyed persistent resources per ``(ubatch, lane)``.
Can be locked to prevent growth or new resources during execution.
"""
def __init__(
@@ -66,6 +68,9 @@ class WorkspaceManager:
self._current_workspaces: list[torch.Tensor | None] = [None] * (
self._num_ubatches * self._num_lanes
)
self._persistent_resources: list[dict[Hashable, object]] = [
{} for _ in self._current_workspaces
]
self._locked: bool = False
@staticmethod
@@ -76,10 +81,10 @@ class WorkspaceManager:
return workspace.numel() * workspace.element_size()
def lock(self) -> None:
"""Lock the workspace to prevent further growth.
"""Lock the workspace to prevent growth or new persistent resources.
After locking, any attempt to allocate a larger workspace will raise
an assertion error. This ensures workspace size is fixed during execution.
Larger scratch requests and new persistent resources raise an assertion
error. Existing resources retain their storage throughout execution.
"""
self._locked = True
if envs.VLLM_DEBUG_WORKSPACE:
@@ -113,6 +118,63 @@ class WorkspaceManager:
"""Check if workspace is locked."""
return self._locked
def _get_workspace_id(self) -> int:
lane = _workspace_lane.get()
if lane >= self._num_lanes:
raise RuntimeError(
f"Workspace lane {lane} is not configured; manager has "
f"{self._num_lanes} lane(s)."
)
return dbo_current_ubatch_id() * self._num_lanes + lane
def get_persistent_resource(self, key: Hashable, factory: Callable[[], T]) -> T:
"""Return the resource cached for ``key`` in the current ubatch and lane.
The first request calls ``factory`` and stores its result for the lifetime
of this manager. Later requests return the same object without calling
the factory again. Each ubatch/lane pair has a separate cache. Once the
manager is locked, requesting a key missing from that cache raises an
AssertionError.
Keys must identify the resource type and configuration. Callers must not
resize or replace tensor storage held by a cached resource. Uses of the
same resource must not overlap; include the CUDA stream in the key when
streams can execute concurrently.
With torch.compile, call this inside a runtime custom op so resource
selection happens during execution.
"""
resources = self._persistent_resources[self._get_workspace_id()]
if key not in resources:
if self._locked:
raise AssertionError(
f"Workspace is locked but {key!r} was not allocated during "
"warmup for the current ubatch and lane."
)
resources[key] = factory()
return cast(T, resources[key])
def get_persistent(
self,
key: Hashable,
shape: tuple[int, ...],
dtype: torch.dtype,
*,
zero_init: bool = False,
) -> torch.Tensor:
"""Get a fixed-shape tensor, optionally zeroed on first allocation only."""
factory = torch.zeros if zero_init else torch.empty
workspace = self.get_persistent_resource(
(torch.Tensor, key),
lambda: factory(shape, dtype=dtype, device=self._device),
)
if workspace.shape != shape or workspace.dtype != dtype:
raise ValueError(
f"Persistent workspace {key!r} has shape={workspace.shape}, "
f"dtype={workspace.dtype}; requested shape={shape}, dtype={dtype}."
)
return workspace
def get_simultaneous(
self, *shapes_and_dtypes: tuple[tuple[int, ...], torch.dtype]
) -> list[torch.Tensor]:
@@ -151,14 +213,7 @@ class WorkspaceManager:
The current workspace tensor.
"""
ubatch_id = dbo_current_ubatch_id()
lane = _workspace_lane.get()
if lane >= self._num_lanes:
raise RuntimeError(
f"Workspace lane {lane} is not configured; manager has "
f"{self._num_lanes} lane(s)."
)
workspace_id = ubatch_id * self._num_lanes + lane
workspace_id = self._get_workspace_id()
current_workspace = self._current_workspaces[workspace_id]
current_size = self._workspace_size_bytes(current_workspace)
@@ -218,8 +273,8 @@ class WorkspaceManager:
get_caller_info(),
current_size / _MB,
required_bytes / _MB,
ubatch_id,
lane,
workspace_id // self._num_lanes,
workspace_id % self._num_lanes,
)
return current_workspace
@@ -277,12 +332,11 @@ def init_workspace_manager(
def lock_workspace() -> None:
"""Lock the workspace to prevent further growth.
"""Lock the workspace to prevent growth or new persistent resources.
After calling this function, any attempt to allocate a workspace larger
than the current size will raise an AssertionError. This ensures that
workspace size is fixed during execution and prevents unexpected memory
allocations in the hot path.
Larger scratch requests and new persistent resources raise an AssertionError.
All required resources must be initialized in each execution slot during
warmup, so their storage remains fixed during execution.
Example:
# During initialization