[Offloader] Stream multimodal towers from host RAM with bulk-copy

Add --offload-towers: place multimodal tower weights in pinned host
memory and bulk-copy each unit to the GPU only during its forward,
keeping towers out of VRAM between encodes. Unlike UVA zero-copy,
whole-tensor H2D transfers avoid re-reading operand tiles over PCIe
inside the GEMM inner loop. Tower streaming is independent of layer
group scheduling, so group_size may be 0 for tower-only offload.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 14:32:14 +07:00
co-authored by Hinata
parent 5d6754ff6b
commit 18c06ed406
5 changed files with 307 additions and 14 deletions
+204 -5
View File
@@ -150,6 +150,10 @@ class PrefetchOffloader(BaseOffloader):
group_size: Group every N layers together.
num_in_group: Offload this many layers per group (last N of each group).
prefetch_step: Number of layers to prefetch ahead.
offload_towers: Also offload multimodal tower weights to pinned host
memory, streaming them to the GPU during each encoder forward
(bulk-copy). Group scheduling still applies to decoder layers
only; `group_size` may be 0 when only towers are offloaded.
mode: Offload mode ("cpu" is currently supported).
"""
@@ -160,19 +164,24 @@ class PrefetchOffloader(BaseOffloader):
num_in_group: int,
prefetch_step: int,
offload_params: set[str] | None = None,
offload_towers: bool = False,
mode: str = "cpu",
):
self.group_size = group_size
self.num_in_group = num_in_group
self.prefetch_step = prefetch_step
self.offload_params = offload_params or set()
self.offload_towers = offload_towers
self.mode = mode
if offload_towers:
self.supports_tower_offload = True
# Copy stream for async H2D transfers
self.copy_stream = torch.cuda.Stream()
# Module offloaders and buffer pool (populated in wrap_modules/post_init)
self.module_offloaders: list[_ModuleOffloader] = []
self.tower_streamers: list[_TowerUnitStreamer] = []
self.buffer_pool: StaticBufferPool | None = None
self.total_offloaded_bytes = 0
@@ -182,17 +191,29 @@ class PrefetchOffloader(BaseOffloader):
prefix: str = "",
) -> list[nn.Module]:
"""Wrap modules with prefetch offloading logic."""
assert len(self.module_offloaders) == 0, (
"wrap_modules should only be called once"
)
modules = list(modules_generator)
if prefix:
prefix = f"{prefix}."
# Routed by _mark_tower_model: a whole tower module, handled by
# the streaming path, independent of the decoder group schedule.
if self.offload_towers:
for module in modules:
self._wrap_tower(module, prefix)
return modules
if self.group_size <= 0:
# Prefetch backend selected for towers only; decoder layers
# stay resident on the device.
return modules
assert len(self.module_offloaders) == 0, (
"wrap_modules should only be called once for the decoder stack"
)
all_modules = []
offload_modules = []
for module_index, module in enumerate(modules_generator):
for module_index, module in enumerate(modules):
all_modules.append(module)
# Select layers to offload based on group pattern
@@ -228,6 +249,39 @@ class PrefetchOffloader(BaseOffloader):
return all_modules
def _wrap_tower(self, tower: nn.Module, prefix: str):
"""Split a tower into parameter-bearing units and place each one
in pinned host RAM until its forward executes (bulk-copy).
"""
first_param = next(tower.parameters(), None)
if first_param is None:
# Tower was staged out (e.g. --limit-mm-per-prompt zero for all
# its modalities) or carries no parameters.
return
device = first_param.device
if device.type == "cpu":
return
num_units = 0
for submodule in tower.modules():
param_names = [
name for name, _ in submodule.named_parameters(recurse=False)
]
if not param_names:
continue
streamer = _TowerUnitStreamer(submodule, param_names, device)
streamer.offload_to_host_now()
self.tower_streamers.append(streamer)
num_units += 1
logger.info(
"[PrefetchOffloader] Queued tower '%s' for host streaming: "
"%d parameter units (weights move to the GPU only during "
"encoder forwards).",
prefix,
num_units,
)
def _hook_module_forward(self, index: int, module: nn.Module):
"""Hook module's forward with torch.compile-compatible sync."""
original_forward = module.forward
@@ -340,6 +394,21 @@ class PrefetchOffloader(BaseOffloader):
(in _CpuParamOffloader.__init__), so GPU memory is available for the
static buffer pool.
"""
# Tower streamers: re-snapshot host storage from the final processed
# weights, then install the streaming forwards.
for streamer in self.tower_streamers:
streamer.sync_storage()
for streamer in self.tower_streamers:
streamer.hook_forward()
if self.tower_streamers:
tower_bytes = sum(s.offloaded_bytes for s in self.tower_streamers)
self.total_offloaded_bytes += tower_bytes
logger.info_once(
f"[PrefetchOffloader] Tower offload active: "
f"{len(self.tower_streamers)} units, "
f"{tower_bytes / 1e9:.4f} GB in pinned host RAM."
)
# Sync CPU storage with current param.data BEFORE collecting param info.
# This is needed because process_weights_after_loading may have:
# 1. Transformed weights (quantization, transpose, etc.)
@@ -391,6 +460,136 @@ class PrefetchOffloader(BaseOffloader):
self.module_offloaders[i].start_onload_to_static()
class _TowerUnitStreamer:
"""Bulk-copy placement for one parameter-bearing submodule of a tower.
The unit's parameters live in pinned host memory between forwards.
On each forward, they are bulk-copied to freshly allocated GPU tensors
on the compute stream (so copies are stream-ordered ahead of the unit's
kernels with no explicit sync) and released right after. At steady
state only the unit currently executing holds VRAM, and freed blocks
are recycled by the caching allocator for the next unit's weights.
This is the bulk-copy alternative to UVA zero-copy for multimodal
towers: whole-tensor H2D transfers instead of operand tiles re-read
over PCIe inside the GEMM inner loop.
Interaction notes:
- If an encoder CUDA graph captures a streamed unit, correctness is
preserved (the memcpy is captured and replays from host memory), but
the captured allocations persist in the graph pool, so the VRAM
saving is lost. Prefer eager encoder execution (`cudagraph_mm_encoder`
off) together with tower offloading.
- Sleep/wake weight reload is not intercepted; storage is re-synced
from `param.data` at `post_init` only.
"""
def __init__(self, unit: nn.Module, param_names: list[str], device: Any):
self.unit = unit
self.param_names = param_names
self.device = device
self.offloaded_bytes = 0
# name -> pinned host tensor holding the weights between forwards.
# Populated/refreshed in sync_storage() (after weight loading and
# process_weights_after_loading), which is the authoritative point.
self._storages: dict[str, torch.Tensor] = {}
def _get_param(self, name: str) -> nn.Parameter:
return getattr(self.unit, name)
@staticmethod
def _empty_cpu_like(data: torch.Tensor, pin_memory: bool) -> torch.Tensor:
return torch.empty_strided(
size=data.size(),
stride=data.stride(),
dtype=data.dtype,
layout=data.layout,
device="cpu",
pin_memory=pin_memory,
)
def offload_to_host_now(self) -> None:
"""Move current parameter data to pinned host storage during
construction, so weight loading writes to host memory and the
device never accumulates tower weights.
"""
pin_memory = should_pin_memory()
for name in self.param_names:
p = self._get_param(name)
if p.device.type == "cpu":
continue
cpu_data = self._empty_cpu_like(p.data, pin_memory)
cpu_data.copy_(p.data)
p.data = cpu_data
def sync_storage(self) -> None:
"""Re-snapshot host storage from param.data.
Called from post_init() after weight loading and
process_weights_after_loading. device_loading_context may have
replaced param.data with fresh (already pinned) CPU tensors, and
quantization processing may have left data on the device; handle
both so _storages always references the final processed weights.
"""
self.offloaded_bytes = 0
for name in list(self.param_names):
try:
p = self._get_param(name)
except AttributeError:
# Parameter deleted by process_weights_after_loading.
self.param_names.remove(name)
continue
data = p.data
if data.device.type != "cpu":
cpu_data = self._empty_cpu_like(data, pin_memory=should_pin_memory())
cpu_data.copy_(data)
data = cpu_data
p.data = data
elif should_pin_memory() and not data.is_pinned():
pinned = self._empty_cpu_like(data, pin_memory=True)
pinned.copy_(data)
data = pinned
p.data = data
self._storages[name] = data
self.offloaded_bytes += data.numel() * data.element_size()
def hook_forward(self) -> None:
if getattr(self.unit, "_vllm_tower_streamed", False):
return # post_init ran more than once
original_forward = self.unit.forward
streamer = self
def forward(*args, **kwargs):
if torch.cuda.is_current_stream_capturing():
logger.warning_once(
"[_TowerUnitStreamer] Streaming a tower unit during "
"CUDA graph capture: results stay correct but the "
"weights will be captured into the graph pool, so no "
"VRAM is saved. Disable encoder CUDA graphs "
"(cudagraph_mm_encoder) to keep the saving."
)
for name in streamer.param_names:
storage = streamer._storages.get(name)
if storage is None:
continue
# Async H2D on the compute stream: ordered ahead of this
# unit's kernels, no explicit synchronization needed.
streamer._get_param(name).data = storage.to(
streamer.device, non_blocking=True
)
try:
return original_forward(*args, **kwargs)
finally:
for name in streamer.param_names:
storage = streamer._storages.get(name)
if storage is None:
continue
streamer._get_param(name).data = storage
self.unit._vllm_tower_streamed = True
self.unit.forward = forward
class _ModuleOffloader:
"""Manages offloading for a single module.