[Offloader] Offload submodules that make_layers never reaches (#53120)

Signed-off-by: ray24777 <103923677+ray24777@users.noreply.github.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ray24777
2026-08-26 18:37:00 +08:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7a9993878c
commit 61d4f56635
5 changed files with 113 additions and 4 deletions
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from operator import attrgetter
import pytest
import torch.nn as nn
import vllm.envs as envs
from vllm.model_executor.offloader import (
@@ -82,3 +85,59 @@ def test_mrv2_weight_offloading(
finally:
set_offloader(original_offloader)
envs.disable_envs_cache()
def _is_offloaded(p: nn.Parameter) -> bool:
return p.device.type == "cpu" or getattr(p, "_vllm_is_uva_offloaded", False)
@pytest.mark.parametrize("disable_uva", [False, True])
def test_tower_weight_offloading(vllm_runner, monkeypatch, disable_uva):
"""`cpu_offload_params` segments must reach towers built outside make_layers.
Regression test: `wrap_modules` was only called from `make_layers`, so a
directly-constructed vision tower never reached the offloader and segments
targeting it silently matched nothing.
"""
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
if disable_uva:
monkeypatch.setenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "1")
envs.disable_envs_cache()
original_offloader = get_offloader()
try:
with vllm_runner(
"Qwen/Qwen3.5-0.8B",
enforce_eager=True,
# allocate more vram as Qwen 3.5 has 1.6 GiB of weights
gpu_memory_utilization=0.3,
max_model_len=128,
max_num_seqs=1,
enable_prefix_caching=False,
cpu_offload_gb=1,
cpu_offload_params={"visual"},
) as vllm_model:
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
offloader = get_offloader()
assert isinstance(offloader, UVAOffloader)
assert offloader.cpu_offload_bytes > 0
model = model_runner.get_model()
assert model._tower_model_names
for name in model._tower_model_names:
tower = attrgetter(name)(model)
params = list(tower.parameters())
assert params
assert all(_is_offloaded(p) for p in params)
if disable_uva:
# non-UVA fallback: weights live on CPU and are moved
# back on first forward
assert "forward" in vars(tower)
# The language model must stay resident.
assert not any(_is_offloaded(p) for p in model.language_model.parameters())
finally:
set_offloader(original_offloader)
envs.disable_envs_cache()