[MRv2 Feature] MR v2 weight offloading support (#51413)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -8,8 +8,10 @@ from ..utils import compare_two_settings
|
|||||||
|
|
||||||
@pytest.mark.parametrize("disable_pin_memory", [False, True])
|
@pytest.mark.parametrize("disable_pin_memory", [False, True])
|
||||||
@pytest.mark.parametrize("disable_uva", [False, True])
|
@pytest.mark.parametrize("disable_uva", [False, True])
|
||||||
def test_cpu_offload(disable_pin_memory, disable_uva):
|
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
|
||||||
|
def test_cpu_offload(disable_pin_memory, disable_uva, use_v2_model_runner):
|
||||||
env_vars = {
|
env_vars = {
|
||||||
|
"VLLM_USE_V2_MODEL_RUNNER": str(int(use_v2_model_runner)),
|
||||||
"VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY": str(int(disable_pin_memory)),
|
"VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY": str(int(disable_pin_memory)),
|
||||||
"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": str(int(disable_uva)),
|
"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": str(int(disable_uva)),
|
||||||
}
|
}
|
||||||
@@ -24,6 +26,6 @@ def test_cpu_offload(disable_pin_memory, disable_uva):
|
|||||||
model="hmellor/tiny-random-LlamaForCausalLM",
|
model="hmellor/tiny-random-LlamaForCausalLM",
|
||||||
arg1=[],
|
arg1=[],
|
||||||
arg2=args,
|
arg2=args,
|
||||||
env1=None,
|
env1=env_vars,
|
||||||
env2=env_vars,
|
env2=env_vars,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -2,10 +2,13 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Test prefetch offloading correctness with Llama model."""
|
"""Test prefetch offloading correctness with Llama model."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from ..utils import compare_two_settings
|
from ..utils import compare_two_settings
|
||||||
|
|
||||||
|
|
||||||
def test_prefetch_offload_llama():
|
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
|
||||||
|
def test_prefetch_offload_llama(use_v2_model_runner):
|
||||||
"""Test prefetch CPU offloading with Llama-3.2-1B-Instruct.
|
"""Test prefetch CPU offloading with Llama-3.2-1B-Instruct.
|
||||||
|
|
||||||
Compares outputs between:
|
Compares outputs between:
|
||||||
@@ -30,4 +33,6 @@ def test_prefetch_offload_llama():
|
|||||||
"down_proj",
|
"down_proj",
|
||||||
],
|
],
|
||||||
[], # Baseline: no offloading
|
[], # Baseline: no offloading
|
||||||
|
env1={"VLLM_USE_V2_MODEL_RUNNER": str(int(use_v2_model_runner))},
|
||||||
|
env2={"VLLM_USE_V2_MODEL_RUNNER": str(int(use_v2_model_runner))},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -46,6 +46,11 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
|||||||
initialize_mamba_ssu_backend,
|
initialize_mamba_ssu_backend,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.model_loader import get_model_loader
|
from vllm.model_executor.model_loader import get_model_loader
|
||||||
|
from vllm.model_executor.offloader import (
|
||||||
|
create_offloader,
|
||||||
|
get_offloader,
|
||||||
|
set_offloader,
|
||||||
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.encoder_budget import (
|
from vllm.multimodal.encoder_budget import (
|
||||||
MultiModalBudget,
|
MultiModalBudget,
|
||||||
@@ -292,6 +297,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
self.eplb = EPLBController(self.parallel_config, self.device)
|
self.eplb = EPLBController(self.parallel_config, self.device)
|
||||||
self.routed_experts_capturer: RoutedExpertsCapturer | None = None
|
self.routed_experts_capturer: RoutedExpertsCapturer | None = None
|
||||||
|
|
||||||
|
set_offloader(create_offloader(self.vllm_config.offload_config))
|
||||||
|
|
||||||
def update_max_model_len(self, max_model_len: int) -> None:
|
def update_max_model_len(self, max_model_len: int) -> None:
|
||||||
self.max_model_len = max_model_len
|
self.max_model_len = max_model_len
|
||||||
self.req_states.max_model_len = max_model_len
|
self.req_states.max_model_len = max_model_len
|
||||||
@@ -413,6 +420,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
device=self.device,
|
device=self.device,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
get_offloader().post_init()
|
||||||
|
|
||||||
def get_model(self) -> nn.Module:
|
def get_model(self) -> nn.Module:
|
||||||
return self.model
|
return self.model
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user