[Offloader] Offload submodules that make_layers never reaches (#53120)
Signed-off-by: ray24777 <103923677+ray24777@users.noreply.github.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7a9993878c
commit
61d4f56635
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from operator import attrgetter
|
||||
|
||||
import pytest
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.offloader import (
|
||||
@@ -82,3 +85,59 @@ def test_mrv2_weight_offloading(
|
||||
finally:
|
||||
set_offloader(original_offloader)
|
||||
envs.disable_envs_cache()
|
||||
|
||||
|
||||
def _is_offloaded(p: nn.Parameter) -> bool:
|
||||
return p.device.type == "cpu" or getattr(p, "_vllm_is_uva_offloaded", False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disable_uva", [False, True])
|
||||
def test_tower_weight_offloading(vllm_runner, monkeypatch, disable_uva):
|
||||
"""`cpu_offload_params` segments must reach towers built outside make_layers.
|
||||
|
||||
Regression test: `wrap_modules` was only called from `make_layers`, so a
|
||||
directly-constructed vision tower never reached the offloader and segments
|
||||
targeting it silently matched nothing.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
if disable_uva:
|
||||
monkeypatch.setenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "1")
|
||||
envs.disable_envs_cache()
|
||||
original_offloader = get_offloader()
|
||||
|
||||
try:
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
enforce_eager=True,
|
||||
# allocate more vram as Qwen 3.5 has 1.6 GiB of weights
|
||||
gpu_memory_utilization=0.3,
|
||||
max_model_len=128,
|
||||
max_num_seqs=1,
|
||||
enable_prefix_caching=False,
|
||||
cpu_offload_gb=1,
|
||||
cpu_offload_params={"visual"},
|
||||
) as vllm_model:
|
||||
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
|
||||
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
|
||||
|
||||
offloader = get_offloader()
|
||||
assert isinstance(offloader, UVAOffloader)
|
||||
assert offloader.cpu_offload_bytes > 0
|
||||
|
||||
model = model_runner.get_model()
|
||||
assert model._tower_model_names
|
||||
for name in model._tower_model_names:
|
||||
tower = attrgetter(name)(model)
|
||||
params = list(tower.parameters())
|
||||
assert params
|
||||
assert all(_is_offloaded(p) for p in params)
|
||||
if disable_uva:
|
||||
# non-UVA fallback: weights live on CPU and are moved
|
||||
# back on first forward
|
||||
assert "forward" in vars(tower)
|
||||
|
||||
# The language model must stay resident.
|
||||
assert not any(_is_offloaded(p) for p in model.language_model.parameters())
|
||||
finally:
|
||||
set_offloader(original_offloader)
|
||||
envs.disable_envs_cache()
|
||||
|
||||
Reference in New Issue
Block a user