[MRv2 Feature] MR v2 weight offloading support (#51413)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -46,6 +46,11 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.offloader import (
|
||||
create_offloader,
|
||||
get_offloader,
|
||||
set_offloader,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.encoder_budget import (
|
||||
MultiModalBudget,
|
||||
@@ -292,6 +297,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.eplb = EPLBController(self.parallel_config, self.device)
|
||||
self.routed_experts_capturer: RoutedExpertsCapturer | None = None
|
||||
|
||||
set_offloader(create_offloader(self.vllm_config.offload_config))
|
||||
|
||||
def update_max_model_len(self, max_model_len: int) -> None:
|
||||
self.max_model_len = max_model_len
|
||||
self.req_states.max_model_len = max_model_len
|
||||
@@ -413,6 +420,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
get_offloader().post_init()
|
||||
|
||||
def get_model(self) -> nn.Module:
|
||||
return self.model
|
||||
|
||||
|
||||
Reference in New Issue
Block a user