[Offloader] Stream multimodal towers from host RAM with bulk-copy

Add --offload-towers: place multimodal tower weights in pinned host
memory and bulk-copy each unit to the GPU only during its forward,
keeping towers out of VRAM between encodes. Unlike UVA zero-copy,
whole-tensor H2D transfers avoid re-reading operand tiles over PCIe
inside the GEMM inner loop. Tower streaming is independent of layer
group scheduling, so group_size may be 0 for tower-only offload.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 14:32:14 +07:00
co-authored by Hinata
parent 5d6754ff6b
commit 18c06ed406
5 changed files with 307 additions and 14 deletions
+5
View File
@@ -604,6 +604,7 @@ class EngineArgs:
offload_num_in_group: int = PrefetchOffloadConfig.offload_num_in_group
offload_prefetch_step: int = PrefetchOffloadConfig.offload_prefetch_step
offload_params: set[str] = get_field(PrefetchOffloadConfig, "offload_params")
offload_towers: bool = PrefetchOffloadConfig.offload_towers
gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
max_num_batched_tokens: int | None = None
@@ -1470,6 +1471,9 @@ class EngineArgs:
offload_group.add_argument(
"--offload-params", **prefetch_kwargs["offload_params"]
)
offload_group.add_argument(
"--offload-towers", **prefetch_kwargs["offload_towers"]
)
# Multimodal related configs
multimodal_kwargs = get_kwargs(MultiModalConfig)
@@ -2839,6 +2843,7 @@ class EngineArgs:
offload_num_in_group=self.offload_num_in_group,
offload_prefetch_step=self.offload_prefetch_step,
offload_params=self.offload_params,
offload_towers=self.offload_towers,
),
)