[Offloader] Stream multimodal towers from host RAM with bulk-copy
Add --offload-towers: place multimodal tower weights in pinned host memory and bulk-copy each unit to the GPU only during its forward, keeping towers out of VRAM between encodes. Unlike UVA zero-copy, whole-tensor H2D transfers avoid re-reading operand tiles over PCIe inside the GEMM inner loop. Tower streaming is independent of layer group scheduling, so group_size may be 0 for tower-only offload. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -56,8 +56,9 @@ class BaseOffloader(ABC):
|
||||
`SupportsMultiModal._mark_tower_model`, outside the `make_layers` call.
|
||||
|
||||
Offloaders whose `wrap_modules` may only be called on the decoder layer
|
||||
stack (e.g. `PrefetchOffloader`, which schedules prefetches over a
|
||||
circular layer stack) must keep this `False`.
|
||||
stack must keep this `False`. `PrefetchOffloader` enables it per-instance
|
||||
when `offload_towers` is configured, via a tower path that does not use
|
||||
the circular layer stack or the static buffer pool.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
@@ -143,8 +144,8 @@ def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
|
||||
"""Create an offloader based on the offload configuration.
|
||||
|
||||
Uses the explicit ``offload_backend`` selector. When set to ``"auto"``,
|
||||
selects prefetch if ``offload_group_size > 0``, UVA if
|
||||
``cpu_offload_gb > 0``, otherwise noop.
|
||||
selects prefetch if ``offload_group_size > 0`` or ``offload_towers`` is
|
||||
set, UVA if ``cpu_offload_gb > 0``, otherwise noop.
|
||||
"""
|
||||
from vllm.model_executor.offloader.prefetch import PrefetchOffloader
|
||||
from vllm.model_executor.offloader.uva import UVAOffloader
|
||||
@@ -154,7 +155,7 @@ def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
|
||||
prefetch = offload_config.prefetch
|
||||
|
||||
if backend == "auto":
|
||||
if prefetch.offload_group_size > 0:
|
||||
if prefetch.offload_group_size > 0 or prefetch.offload_towers:
|
||||
backend = "prefetch"
|
||||
elif uva.cpu_offload_gb > 0:
|
||||
backend = "uva"
|
||||
@@ -167,6 +168,7 @@ def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
|
||||
num_in_group=prefetch.offload_num_in_group,
|
||||
prefetch_step=prefetch.offload_prefetch_step,
|
||||
offload_params=prefetch.offload_params,
|
||||
offload_towers=prefetch.offload_towers,
|
||||
mode="cpu",
|
||||
)
|
||||
elif backend == "uva":
|
||||
|
||||
Reference in New Issue
Block a user