[Offloader] Stream multimodal towers from host RAM with bulk-copy

Add --offload-towers: place multimodal tower weights in pinned host
memory and bulk-copy each unit to the GPU only during its forward,
keeping towers out of VRAM between encodes. Unlike UVA zero-copy,
whole-tensor H2D transfers avoid re-reading operand tiles over PCIe
inside the GEMM inner loop. Tower streaming is independent of layer
group scheduling, so group_size may be 0 for tower-only offload.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 14:32:14 +07:00
co-authored by Hinata
parent 5d6754ff6b
commit 18c06ed406
5 changed files with 307 additions and 14 deletions
+7 -5
View File
@@ -56,8 +56,9 @@ class BaseOffloader(ABC):
`SupportsMultiModal._mark_tower_model`, outside the `make_layers` call.
Offloaders whose `wrap_modules` may only be called on the decoder layer
stack (e.g. `PrefetchOffloader`, which schedules prefetches over a
circular layer stack) must keep this `False`.
stack must keep this `False`. `PrefetchOffloader` enables it per-instance
when `offload_towers` is configured, via a tower path that does not use
the circular layer stack or the static buffer pool.
"""
@abstractmethod
@@ -143,8 +144,8 @@ def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
"""Create an offloader based on the offload configuration.
Uses the explicit ``offload_backend`` selector. When set to ``"auto"``,
selects prefetch if ``offload_group_size > 0``, UVA if
``cpu_offload_gb > 0``, otherwise noop.
selects prefetch if ``offload_group_size > 0`` or ``offload_towers`` is
set, UVA if ``cpu_offload_gb > 0``, otherwise noop.
"""
from vllm.model_executor.offloader.prefetch import PrefetchOffloader
from vllm.model_executor.offloader.uva import UVAOffloader
@@ -154,7 +155,7 @@ def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
prefetch = offload_config.prefetch
if backend == "auto":
if prefetch.offload_group_size > 0:
if prefetch.offload_group_size > 0 or prefetch.offload_towers:
backend = "prefetch"
elif uva.cpu_offload_gb > 0:
backend = "uva"
@@ -167,6 +168,7 @@ def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
num_in_group=prefetch.offload_num_in_group,
prefetch_step=prefetch.offload_prefetch_step,
offload_params=prefetch.offload_params,
offload_towers=prefetch.offload_towers,
mode="cpu",
)
elif backend == "uva":