[Offloader] Stream multimodal towers from host RAM with bulk-copy
Add --offload-towers: place multimodal tower weights in pinned host memory and bulk-copy each unit to the GPU only during its forward, keeping towers out of VRAM between encodes. Unlike UVA zero-copy, whole-tensor H2D transfers avoid re-reading operand tiles over PCIe inside the GEMM inner loop. Tower streaming is independent of layer group scheduling, so group_size may be 0 for tower-only offload. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -604,6 +604,7 @@ class EngineArgs:
|
||||
offload_num_in_group: int = PrefetchOffloadConfig.offload_num_in_group
|
||||
offload_prefetch_step: int = PrefetchOffloadConfig.offload_prefetch_step
|
||||
offload_params: set[str] = get_field(PrefetchOffloadConfig, "offload_params")
|
||||
offload_towers: bool = PrefetchOffloadConfig.offload_towers
|
||||
gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
|
||||
kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
|
||||
max_num_batched_tokens: int | None = None
|
||||
@@ -1470,6 +1471,9 @@ class EngineArgs:
|
||||
offload_group.add_argument(
|
||||
"--offload-params", **prefetch_kwargs["offload_params"]
|
||||
)
|
||||
offload_group.add_argument(
|
||||
"--offload-towers", **prefetch_kwargs["offload_towers"]
|
||||
)
|
||||
|
||||
# Multimodal related configs
|
||||
multimodal_kwargs = get_kwargs(MultiModalConfig)
|
||||
@@ -2839,6 +2843,7 @@ class EngineArgs:
|
||||
offload_num_in_group=self.offload_num_in_group,
|
||||
offload_prefetch_step=self.offload_prefetch_step,
|
||||
offload_params=self.offload_params,
|
||||
offload_towers=self.offload_towers,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user