[MyPy] Fix mypy for vllm/lora (#41722)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-06-22 10:57:09 -04:00
committed by GitHub
parent 1c7bc18318
commit ccd49f6821
15 changed files with 140 additions and 50 deletions
+21 -8
View File
@@ -124,6 +124,7 @@ def test_replace_submodules(default_vllm_config, dist_init, dummy_model):
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
), ),
torch.device(DEVICES[0]), torch.device(DEVICES[0]),
default_vllm_config,
) )
model = manager.model model = manager.model
assert isinstance(model.get_submodule("dense1"), ColumnParallelLinearWithLoRA) assert isinstance(model.get_submodule("dense1"), ColumnParallelLinearWithLoRA)
@@ -152,6 +153,7 @@ def test_wrap_replicated_linear_subclasses(default_vllm_config, dist_init, dummy
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
), ),
torch.device(DEVICES[0]), torch.device(DEVICES[0]),
default_vllm_config,
) )
assert isinstance( assert isinstance(
@@ -172,6 +174,7 @@ def test_wrap_gate_linear(default_vllm_config, dist_init, dummy_model):
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
), ),
torch.device(DEVICES[0]), torch.device(DEVICES[0]),
default_vllm_config,
) )
assert isinstance( assert isinstance(
@@ -219,6 +222,7 @@ def test_dedup_shared_module_across_paths(default_vllm_config, dist_init, dummy_
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
), ),
torch.device(DEVICES[0]), torch.device(DEVICES[0]),
default_vllm_config,
) )
canonical = manager.model.get_submodule("moe.gate") canonical = manager.model.get_submodule("moe.gate")
@@ -263,6 +267,7 @@ def test_lm_head_exempt_from_dedup(default_vllm_config, dist_init, dummy_model):
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
), ),
torch.device(DEVICES[0]), torch.device(DEVICES[0]),
default_vllm_config,
) )
# lm_head's special handling still ran: logits_processor got wrapped # lm_head's special handling still ran: logits_processor got wrapped
@@ -293,6 +298,7 @@ def test_skip_unsupported_matched_modules(default_vllm_config, dist_init, dummy_
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
), ),
torch.device(DEVICES[0]), torch.device(DEVICES[0]),
default_vllm_config,
) )
# Should not crash and should keep unsupported matched modules unchanged. # Should not crash and should keep unsupported matched modules unchanged.
@@ -325,6 +331,7 @@ def test_target_modules_fail_closed_on_unsupported_matched_modules(
target_modules=["dense1"], target_modules=["dense1"],
), ),
torch.device(DEVICES[0]), torch.device(DEVICES[0]),
default_vllm_config,
) )
@@ -374,6 +381,7 @@ def test_lora_model_manager(default_vllm_config, dist_init, dummy_model, device)
max_lora_rank=8, max_cpu_loras=3, max_loras=2, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=3, max_loras=2, lora_dtype=DEFAULT_DTYPE
), ),
device=device, device=device,
vllm_config=default_vllm_config,
) )
assert all(x is None for x in manager.lora_index_to_id) assert all(x is None for x in manager.lora_index_to_id)
assert manager.add_adapter(model_lora1) assert manager.add_adapter(model_lora1)
@@ -442,6 +450,7 @@ def test_lora_lru_cache_model_manager(
max_lora_rank=8, max_cpu_loras=3, max_loras=2, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=3, max_loras=2, lora_dtype=DEFAULT_DTYPE
), ),
device=device, device=device,
vllm_config=default_vllm_config,
) )
assert all(x is None for x in manager.lora_index_to_id) assert all(x is None for x in manager.lora_index_to_id)
assert manager.add_adapter(model_lora1) assert manager.add_adapter(model_lora1)
@@ -535,6 +544,7 @@ def test_lru_lora_model_manager(default_vllm_config, dist_init, dummy_model, dev
max_lora_rank=8, max_cpu_loras=2, max_loras=2, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=2, max_loras=2, lora_dtype=DEFAULT_DTYPE
), ),
device=device, device=device,
vllm_config=default_vllm_config,
) )
assert all(x is None for x in manager.lora_index_to_id) assert all(x is None for x in manager.lora_index_to_id)
@@ -642,9 +652,7 @@ def test_lru_lora_model_manager(default_vllm_config, dist_init, dummy_model, dev
@pytest.mark.parametrize("device", DEVICES) @pytest.mark.parametrize("device", DEVICES)
def test_lru_cache_worker_adapter_manager( def test_lru_cache_worker_adapter_manager(dist_init, dummy_model, device, tmp_path):
default_vllm_config, dist_init, dummy_model, device, tmp_path
):
lora_config = LoRAConfig( lora_config = LoRAConfig(
max_lora_rank=8, max_cpu_loras=4, max_loras=4, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=4, max_loras=4, lora_dtype=DEFAULT_DTYPE
) )
@@ -670,7 +678,7 @@ def test_lru_cache_worker_adapter_manager(
worker_adapter_manager.max_num_seqs = 4 worker_adapter_manager.max_num_seqs = 4
worker_adapter_manager.max_num_batched_tokens = 2 worker_adapter_manager.max_num_batched_tokens = 2
worker_adapter_manager.create_lora_manager(dummy_model) worker_adapter_manager.create_lora_manager(dummy_model, vllm_config)
mapping = LoRAMapping([], []) mapping = LoRAMapping([], [])
worker_adapter_manager.set_active_adapters( worker_adapter_manager.set_active_adapters(
@@ -758,9 +766,7 @@ def test_lru_cache_worker_adapter_manager(
@pytest.mark.parametrize("device", DEVICES) @pytest.mark.parametrize("device", DEVICES)
def test_worker_adapter_manager( def test_worker_adapter_manager(dist_init, dummy_model_gate_up, device, tmp_path):
default_vllm_config, dist_init, dummy_model_gate_up, device, tmp_path
):
# Should remove every LoRA not specified in the request. # Should remove every LoRA not specified in the request.
lora_config = LoRAConfig( lora_config = LoRAConfig(
max_lora_rank=8, max_cpu_loras=4, max_loras=4, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=4, max_loras=4, lora_dtype=DEFAULT_DTYPE
@@ -774,7 +780,7 @@ def test_worker_adapter_manager(
worker_adapter_manager = WorkerLoRAManager(vllm_config, device, EMBEDDING_MODULES) worker_adapter_manager = WorkerLoRAManager(vllm_config, device, EMBEDDING_MODULES)
worker_adapter_manager.vocab_size = dummy_model_gate_up.unpadded_vocab_size worker_adapter_manager.vocab_size = dummy_model_gate_up.unpadded_vocab_size
worker_adapter_manager.create_lora_manager(dummy_model_gate_up) worker_adapter_manager.create_lora_manager(dummy_model_gate_up, vllm_config)
dummy_lora_files = f"{tmp_path}/lora_adapter" dummy_lora_files = f"{tmp_path}/lora_adapter"
os.makedirs(dummy_lora_files, exist_ok=True) os.makedirs(dummy_lora_files, exist_ok=True)
@@ -894,6 +900,7 @@ def test_packed_loras(default_vllm_config, dist_init, dummy_model_gate_up, devic
max_lora_rank=8, max_cpu_loras=2, max_loras=2, lora_dtype=DEFAULT_DTYPE max_lora_rank=8, max_cpu_loras=2, max_loras=2, lora_dtype=DEFAULT_DTYPE
), ),
device=device, device=device,
vllm_config=default_vllm_config,
) )
model = manager.model model = manager.model
@@ -944,6 +951,7 @@ def _test_target_modules(
device: str, device: str,
expected_lora: list[tuple[str, type]], expected_lora: list[tuple[str, type]],
expected_no_lora: list[tuple[str, type]], expected_no_lora: list[tuple[str, type]],
vllm_config,
): ):
"""Create a LoRAModelManager and assert which modules have LoRA applied.""" """Create a LoRAModelManager and assert which modules have LoRA applied."""
LoRAModelManager( LoRAModelManager(
@@ -959,6 +967,7 @@ def _test_target_modules(
target_modules=target_modules, target_modules=target_modules,
), ),
device=device, device=device,
vllm_config=vllm_config,
) )
for module_path, lora_cls in expected_lora: for module_path, lora_cls in expected_lora:
assert isinstance(model.get_submodule(module_path), lora_cls) assert isinstance(model.get_submodule(module_path), lora_cls)
@@ -981,6 +990,7 @@ def test_target_modules_config(default_vllm_config, dist_init, dummy_model, devi
("dense2", RowParallelLinearWithLoRA), ("dense2", RowParallelLinearWithLoRA),
("layer1.dense2", RowParallelLinearWithLoRA), ("layer1.dense2", RowParallelLinearWithLoRA),
], ],
vllm_config=default_vllm_config,
) )
@@ -998,6 +1008,7 @@ def test_target_modules_multiple(default_vllm_config, dist_init, dummy_model, de
("layer1.dense2", RowParallelLinearWithLoRA), ("layer1.dense2", RowParallelLinearWithLoRA),
], ],
expected_no_lora=[], expected_no_lora=[],
vllm_config=default_vllm_config,
) )
@@ -1017,6 +1028,7 @@ def test_target_modules_none_uses_all(
("layer1.dense2", RowParallelLinearWithLoRA), ("layer1.dense2", RowParallelLinearWithLoRA),
], ],
expected_no_lora=[], expected_no_lora=[],
vllm_config=default_vllm_config,
) )
@@ -1036,4 +1048,5 @@ def test_target_modules_match_packed_runtime_modules(
("layer1.dense1", ColumnParallelLinearWithLoRA), ("layer1.dense1", ColumnParallelLinearWithLoRA),
("layer1.dense2", RowParallelLinearWithLoRA), ("layer1.dense2", RowParallelLinearWithLoRA),
], ],
vllm_config=default_vllm_config,
) )
-2
View File
@@ -25,8 +25,6 @@ import regex as re
# from "skip" to "silent", remove its directory from SEPARATE_GROUPS. # from "skip" to "silent", remove its directory from SEPARATE_GROUPS.
SEPARATE_GROUPS = [ SEPARATE_GROUPS = [
"tests", "tests",
# v0 related
"vllm/lora",
] ]
# TODO(woosuk): Include the code from Megatron and HuggingFace. # TODO(woosuk): Include the code from Megatron and HuggingFace.
+11 -2
View File
@@ -17,6 +17,7 @@ from vllm.forward_context import (
from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.linear import (
ColumnParallelLinear, ColumnParallelLinear,
LinearBase, LinearBase,
QuantizeMethodBase,
ReplicatedLinear, ReplicatedLinear,
RowParallelLinear, RowParallelLinear,
) )
@@ -182,6 +183,14 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
lora_b, non_blocking=True lora_b, non_blocking=True
) )
def _get_quant_method(self) -> QuantizeMethodBase:
quant_method = self.base_layer.quant_method
if quant_method is None:
raise RuntimeError(
f"{type(self.base_layer).__name__} must define quant_method for LoRA."
)
return quant_method
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
# is_forward_context_available for tower modules # is_forward_context_available for tower modules
if self._enable_aux_cuda_stream and is_forward_context_available(): if self._enable_aux_cuda_stream and is_forward_context_available():
@@ -195,7 +204,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
def _apply_sync( def _apply_sync(
self, x: torch.Tensor, bias: torch.Tensor | None = None self, x: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor: ) -> torch.Tensor:
output = self.base_layer.quant_method.apply(self.base_layer, x, bias) output = self._get_quant_method().apply(self.base_layer, x, bias)
return self._apply_lora_to_output(x, output) return self._apply_lora_to_output(x, output)
def _apply_base_forward(self, x: torch.Tensor) -> torch.Tensor: def _apply_base_forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -242,7 +251,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
output_size = sum(self.output_slices) output_size = sum(self.output_slices)
def base_fn() -> torch.Tensor: def base_fn() -> torch.Tensor:
return self.base_layer.quant_method.apply(self.base_layer, x, bias) return self._get_quant_method().apply(self.base_layer, x, bias)
def lora_fn() -> torch.Tensor: def lora_fn() -> torch.Tensor:
# Must be zeros, not empty: _lora_expand_kernel exits early (without # Must be zeros, not empty: _lora_expand_kernel exits early (without
+14 -3
View File
@@ -33,7 +33,7 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
== len(layer.output_slices) == len(layer.output_slices)
) )
output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias) output = layer._get_quant_method().apply(layer.base_layer, x, bias)
x = x.view(-1, x.shape[-1]) x = x.view(-1, x.shape[-1])
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
@@ -73,6 +73,8 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
) )
if not current_platform.can_update_inplace(): if not current_platform.can_update_inplace():
if lora_output is None:
raise RuntimeError("LoRA expand must return an output tensor.")
output = lora_output output = lora_output
output = output.view(*out_orig_shape) output = output.view(*out_orig_shape)
@@ -327,12 +329,16 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear) merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear)
base_forward = getattr(type(self.base_layer), "forward", None)
merged_forward = getattr(merged_cls, "forward", None)
# Effectively unsharded subclasses can safely reuse their custom # Effectively unsharded subclasses can safely reuse their custom
# forward() implementation before applying the LoRA delta. # forward() implementation before applying the LoRA delta.
if ( if (
self.tp_size == 1 self.tp_size == 1
and type(self.base_layer) is not merged_cls and type(self.base_layer) is not merged_cls
and type(self.base_layer).forward is not merged_cls.forward and base_forward is not None
and merged_forward is not None
and base_forward is not merged_forward
): ):
return self._apply_base_forward(x) return self._apply_base_forward(x)
return _mcp_apply(x, bias, self) return _mcp_apply(x, bias, self)
@@ -482,6 +488,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
lora_config: LoRAConfig, lora_config: LoRAConfig,
packed_modules_list: list, packed_modules_list: list,
model_config: PretrainedConfig | None = None, model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool: ) -> bool:
return ( return (
type(source_layer) is maybe_get_oot_by_class(QKVParallelLinear) type(source_layer) is maybe_get_oot_by_class(QKVParallelLinear)
@@ -523,6 +530,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
lora_config: LoRAConfig, lora_config: LoRAConfig,
packed_modules_list: list, packed_modules_list: list,
model_config: PretrainedConfig | None = None, model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool: ) -> bool:
# specifying kwargs so they can be easily accessed in decorator # specifying kwargs so they can be easily accessed in decorator
return super().can_replace_layer( return super().can_replace_layer(
@@ -565,6 +573,7 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo
lora_config: LoRAConfig, lora_config: LoRAConfig,
packed_modules_list: list, packed_modules_list: list,
model_config: PretrainedConfig | None = None, model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool: ) -> bool:
# specifying kwargs so they can be easily accessed in decorator # specifying kwargs so they can be easily accessed in decorator
return super().can_replace_layer( return super().can_replace_layer(
@@ -650,6 +659,7 @@ class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
lora_config: LoRAConfig, lora_config: LoRAConfig,
packed_modules_list: list, packed_modules_list: list,
model_config: PretrainedConfig | None = None, model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool: ) -> bool:
# specifying kwargs so they can be easily accessed in decorator # specifying kwargs so they can be easily accessed in decorator
return super().can_replace_layer( return super().can_replace_layer(
@@ -678,6 +688,7 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
lora_config: LoRAConfig, lora_config: LoRAConfig,
packed_modules_list: list, packed_modules_list: list,
model_config: PretrainedConfig | None = None, model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool: ) -> bool:
# Support MergedColumnParallelLinear with 3 or more slices # Support MergedColumnParallelLinear with 3 or more slices
# (2 slices are handled by MergedColumnParallelLinearWithLoRA) # (2 slices are handled by MergedColumnParallelLinearWithLoRA)
@@ -727,7 +738,7 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
start_idx = 0 start_idx = 0
for output_size in output_sizes: for output_size in output_sizes:
end_idx = start_idx + output_size end_idx = start_idx + output_size
lora_b_list.append(lora_b[start_idx:end_idx, :]) lora_b_list.append(lora_b[start_idx:end_idx])
start_idx = end_idx start_idx = end_idx
lora_b = lora_b_list lora_b = lora_b_list
+25 -4
View File
@@ -12,10 +12,16 @@ from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.custom_op import maybe_get_oot_by_class from vllm.model_executor.custom_op import maybe_get_oot_by_class
from vllm.model_executor.layers.fused_moe import MoERunner from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
LoRAExpertsMixin,
)
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import ( from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod, FusedMoEModularMethod,
) )
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEKernel,
FusedMoEKernelModularImpl,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import ( from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular, MoEPrepareAndFinalizeNoDPEPModular,
) )
@@ -58,6 +64,13 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
routed_experts._ensure_moe_quant_config_init() routed_experts._ensure_moe_quant_config_init()
if getattr(routed_experts.quant_method, "supports_internal_mk", False): if getattr(routed_experts.quant_method, "supports_internal_mk", False):
moe_kernel = routed_experts.quant_method.moe_kernel moe_kernel = routed_experts.quant_method.moe_kernel
assert moe_kernel is not None, (
"Fused MoE quant method must provide a moe_kernel."
)
# Don't let the kernel own shared experts so the runner can
# overlap them with routed experts via a separate CUDA stream.
assert isinstance(moe_kernel.impl, FusedMoEKernelModularImpl)
moe_kernel.impl.shared_experts = None
else: else:
prepare_finalize = MoEPrepareAndFinalizeNoDPEPModular() prepare_finalize = MoEPrepareAndFinalizeNoDPEPModular()
moe_kernel = FusedMoEKernel( moe_kernel = FusedMoEKernel(
@@ -405,7 +418,11 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
def set_mapping(self, punica_wrapper): def set_mapping(self, punica_wrapper):
super().set_mapping(punica_wrapper) super().set_mapping(punica_wrapper)
lora_context = self._build_lora_context() lora_context = self._build_lora_context()
self._moe_kernel.fused_experts.set_lora_context(lora_context) fused_experts = self._moe_kernel.fused_experts
assert isinstance(fused_experts, LoRAExpertsMixin), (
f"{type(fused_experts).__name__} does not support LoRA context setup."
)
fused_experts.set_lora_context(lora_context)
prepare_finalize = self._moe_kernel.prepare_finalize prepare_finalize = self._moe_kernel.prepare_finalize
if hasattr(prepare_finalize, "set_lora_context"): if hasattr(prepare_finalize, "set_lora_context"):
prepare_finalize.set_lora_context(lora_context) prepare_finalize.set_lora_context(lora_context)
@@ -482,9 +499,13 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
) -> None: ) -> None:
"""Initializes lora matrices.""" """Initializes lora matrices."""
assert isinstance(model_config, PretrainedConfig) if model_config is None:
raise ValueError("model_config must be provided for MoE LoRA.")
architectures = model_config.architectures
if not architectures:
raise ValueError("model_config.architectures must be defined for MoE LoRA.")
self._verify_ep_fs(lora_config) self._verify_ep_fs(lora_config)
self._base_model = model_config.architectures[0] self._base_model = architectures[0]
self.max_loras = lora_config.max_loras self.max_loras = lora_config.max_loras
self.fully_sharded = lora_config.fully_sharded_loras self.fully_sharded = lora_config.fully_sharded_loras
+1 -1
View File
@@ -116,7 +116,7 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
return lora_b return lora_b
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
output = self.base_layer.quant_method.apply(self.base_layer, x, bias) output = self._get_quant_method().apply(self.base_layer, x, bias)
x = x.view(-1, x.shape[-1]) x = x.view(-1, x.shape[-1])
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
+11 -3
View File
@@ -111,19 +111,27 @@ def try_get_optimal_moe_lora_config(
# base MoE weight's block-wise quantization, so block_shape is omitted # base MoE weight's block-wise quantization, so block_shape is omitted
# from the config lookup — the non-quantized branch in get_default_config # from the config lookup — the non-quantized branch in get_default_config
# ignores it anyway. # ignores it anyway.
config = try_get_optimal_moe_config(w1_shape, w2_shape, top_k, dtype, M).copy() raw_config = try_get_optimal_moe_config(w1_shape, w2_shape, top_k, dtype, M)
config: dict[str, int | None] = dict(raw_config)
if op_type in [ if op_type in [
"fused_moe_lora_w13_shrink", "fused_moe_lora_w13_shrink",
"fused_moe_lora_w2_shrink", "fused_moe_lora_w2_shrink",
]: ]:
block_size_n = config.get("BLOCK_SIZE_N")
config["BLOCK_SIZE_N"] = min( config["BLOCK_SIZE_N"] = min(
config.get("BLOCK_SIZE_N", 64), next_power_of_2(rank) block_size_n if block_size_n is not None else 64,
next_power_of_2(rank),
) )
elif op_type in [ elif op_type in [
"fused_moe_lora_w13_expand", "fused_moe_lora_w13_expand",
"fused_moe_lora_w2_expand", "fused_moe_lora_w2_expand",
]: ]:
block_size_k = config.get("BLOCK_SIZE_K")
config["BLOCK_SIZE_K"] = max( config["BLOCK_SIZE_K"] = max(
16, min(config.get("BLOCK_SIZE_K", 32), next_power_of_2(rank)) 16,
min(
block_size_k if block_size_k is not None else 32,
next_power_of_2(rank),
),
) )
return config return config
+4 -3
View File
@@ -245,9 +245,10 @@ class LoRAModel:
from tensorizer import TensorDeserializer from tensorizer import TensorDeserializer
tensorizer_config = TensorizerConfig(**tensorizer_config_dict) tensorizer_config = TensorizerConfig(**tensorizer_config_dict)
lora_tensor_path = os.path.join( tensorizer_dir = tensorizer_config.tensorizer_dir
tensorizer_config.tensorizer_dir, "adapter_model.tensors" if tensorizer_dir is None:
) raise ValueError("tensorizer_dir must be set in tensorizer config.")
lora_tensor_path = os.path.join(tensorizer_dir, "adapter_model.tensors")
tensorizer_args = tensorizer_config._construct_tensorizer_args() tensorizer_args = tensorizer_config._construct_tensorizer_args()
tensors = TensorDeserializer( tensors = TensorDeserializer(
lora_tensor_path, lora_tensor_path,
+17 -10
View File
@@ -34,6 +34,7 @@ from vllm.lora.utils import (
from vllm.model_executor.layers.fused_moe import MoERunner from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.models import ( from vllm.model_executor.models import (
SupportsLoRA, SupportsLoRA,
SupportsMultiModal,
is_pooling_model, is_pooling_model,
supports_multimodal, supports_multimodal,
) )
@@ -50,6 +51,12 @@ T = TypeVar("T")
DEFAULT_LANGUAGE_WRAPPER_KEY = "language_model" DEFAULT_LANGUAGE_WRAPPER_KEY = "language_model"
class SupportsLoRAModel(nn.Module, SupportsLoRA): ...
class SupportsLoRAMultiModalModel(SupportsLoRAModel, SupportsMultiModal): ...
class AdapterLRUCache(LRUCache[int, T]): class AdapterLRUCache(LRUCache[int, T]):
def __init__(self, capacity: int, deactivate_fn: Callable[[int], object]): def __init__(self, capacity: int, deactivate_fn: Callable[[int], object]):
super().__init__(capacity) super().__init__(capacity)
@@ -66,13 +73,13 @@ class LoRAModelManager:
def __init__( def __init__(
self, self,
model: SupportsLoRA, model: SupportsLoRAModel,
max_num_seqs: int, max_num_seqs: int,
max_num_batched_tokens: int, max_num_batched_tokens: int,
vocab_size: int, vocab_size: int,
lora_config: LoRAConfig, lora_config: LoRAConfig,
device: torch.device, device: torch.device,
vllm_config: VllmConfig | None = None, vllm_config: VllmConfig,
): ):
"""Create a LoRAModelManager and adapter for a given model. """Create a LoRAModelManager and adapter for a given model.
@@ -85,7 +92,7 @@ class LoRAModelManager:
vocab_size: the vocab size of the model. vocab_size: the vocab size of the model.
lora_config: the LoRA configuration. lora_config: the LoRA configuration.
""" """
self.model: SupportsLoRA = model self.model: SupportsLoRAModel = model
self.supported_lora_modules = get_supported_lora_modules(self.model) self.supported_lora_modules = get_supported_lora_modules(self.model)
assert self.supported_lora_modules, ( assert self.supported_lora_modules, (
f"No supported LoRA modules found in {self.model.__class__.__name__}." f"No supported LoRA modules found in {self.model.__class__.__name__}."
@@ -106,7 +113,6 @@ class LoRAModelManager:
self.is_pooling_model = is_pooling_model(self.model) self.is_pooling_model = is_pooling_model(self.model)
self.packed_modules: dict[str, list[str]] = {} self.packed_modules: dict[str, list[str]] = {}
self.modules: dict[str, BaseLayerWithLoRA] = {} self.modules: dict[str, BaseLayerWithLoRA] = {}
# Dict instead of a set for compatibility with LRUCache.
self._last_mapping: LoRAMapping | None = None self._last_mapping: LoRAMapping | None = None
is_moe = is_moe_model(self.model) is_moe = is_moe_model(self.model)
self._is_moe = is_moe self._is_moe = is_moe
@@ -272,6 +278,7 @@ class LoRAModelManager:
@property @property
def capacity(self) -> int: def capacity(self) -> int:
assert self.lora_config.max_cpu_loras is not None
return self.lora_config.max_cpu_loras return self.lora_config.max_cpu_loras
@property @property
@@ -1156,7 +1163,7 @@ class LoRAModelManager:
class LoRALRUCache(AdapterLRUCache[LoRAModel]): class LoRALRUCache(AdapterLRUCache[LoRAModel]):
def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int], bool]): def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int], object]):
super().__init__(capacity, deactivate_lora_fn) super().__init__(capacity, deactivate_lora_fn)
@@ -1165,13 +1172,13 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
def __init__( def __init__(
self, self,
model: nn.Module, model: SupportsLoRAModel,
max_num_seqs: int, max_num_seqs: int,
max_num_batched_tokens: int, max_num_batched_tokens: int,
vocab_size: int, vocab_size: int,
lora_config: LoRAConfig, lora_config: LoRAConfig,
device: torch.device, device: torch.device,
vllm_config: VllmConfig | None = None, vllm_config: VllmConfig,
): ):
super().__init__( super().__init__(
model, model,
@@ -1182,10 +1189,10 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
device, device,
vllm_config, vllm_config,
) )
self._registered_adapters: LoRALRUCache = LoRALRUCache( self._registered_adapters: LoRALRUCache = LoRALRUCache( # type: ignore[assignment]
self.capacity, self.deactivate_adapter self.capacity, self.deactivate_adapter
) )
self._active_adapters: LoRALRUCache = LoRALRUCache( self._active_adapters: LoRALRUCache = LoRALRUCache( # type: ignore[assignment]
self.lora_slots, self._deactivate_adapter self.lora_slots, self._deactivate_adapter
) )
@@ -1248,7 +1255,7 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
def create_lora_manager( def create_lora_manager(
model: nn.Module, model: SupportsLoRAModel,
max_num_seqs: int, max_num_seqs: int,
max_num_batched_tokens: int, max_num_batched_tokens: int,
vocab_size: int, vocab_size: int,
+5 -3
View File
@@ -91,9 +91,11 @@ class PEFTHelper:
tensorizer_args = tensorizer_config._construct_tensorizer_args() tensorizer_args = tensorizer_config._construct_tensorizer_args()
from tensorizer.stream_io import open_stream from tensorizer.stream_io import open_stream
lora_config_path = os.path.join( tensorizer_dir = tensorizer_config.tensorizer_dir
tensorizer_config.tensorizer_dir, "adapter_config.json" if tensorizer_dir is None:
) raise ValueError("tensorizer_dir must be set in tensorizer config.")
lora_config_path = os.path.join(tensorizer_dir, "adapter_config.json")
with open_stream( with open_stream(
lora_config_path, mode="rb", **tensorizer_args.stream_kwargs lora_config_path, mode="rb", **tensorizer_args.stream_kwargs
) as f: ) as f:
+16 -5
View File
@@ -173,11 +173,18 @@ def parse_fine_tuned_lora_name(
# mapping correctly. # mapping correctly.
if name.startswith("base_model.model."): if name.startswith("base_model.model."):
name = name.replace("base_model.model.", "") name = name.replace("base_model.model.", "")
name = weights_mapper._map_name(name) if weights_mapper else name if weights_mapper:
# recover the prefix `base_model.model.` mapped_name = weights_mapper._map_name(name)
name = "base_model.model." + name if mapped_name is None:
raise ValueError("Mapped LoRA weight name cannot be None.")
# recover the prefix `base_model.model.`
name = "base_model.model." + mapped_name
else: else:
name = weights_mapper._map_name(name) if weights_mapper else name if weights_mapper:
mapped_name = weights_mapper._map_name(name)
if mapped_name is None:
raise ValueError("Mapped LoRA weight name cannot be None.")
name = mapped_name
# In some situations, we may not start with `base_model.model.`. # In some situations, we may not start with `base_model.model.`.
# If we don't (e.g., ibm-granite/granite-speech-3.3-8b), # If we don't (e.g., ibm-granite/granite-speech-3.3-8b),
@@ -185,7 +192,11 @@ def parse_fine_tuned_lora_name(
start_index = 2 if name.startswith("base_model.model.") else 0 start_index = 2 if name.startswith("base_model.model.") else 0
parts = name.split(".") parts = name.split(".")
if parts[-1] == "weight" and (parts[-2] == "lora_A" or parts[-2] == "lora_B"): if (
parts[-1] == "weight"
and len(parts) >= 2
and (parts[-2] == "lora_A" or parts[-2] == "lora_B")
):
new_name = ".".join(parts[start_index:-2]) new_name = ".".join(parts[start_index:-2])
return new_name, parts[-2] == "lora_A" return new_name, parts[-2] == "lora_A"
+11 -3
View File
@@ -7,6 +7,7 @@ from typing import Any, Literal
import torch import torch
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.lora import LoRAConfig
from vllm.exceptions import LoRAAdapterNotFoundError from vllm.exceptions import LoRAAdapterNotFoundError
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.lora.lora_model import LoRAModel from vllm.lora.lora_model import LoRAModel
@@ -45,7 +46,10 @@ class WorkerLoRAManager:
vllm_config.scheduler_config.max_num_batched_tokens vllm_config.scheduler_config.max_num_batched_tokens
) )
self.vocab_size = vllm_config.model_config.get_vocab_size() self.vocab_size = vllm_config.model_config.get_vocab_size()
self.lora_config = vllm_config.lora_config lora_config = vllm_config.lora_config
if lora_config is None:
raise ValueError("LoRA config must be set for WorkerLoRAManager.")
self.lora_config: LoRAConfig = lora_config
# Use get_text_config() in case of multimodal models # Use get_text_config() in case of multimodal models
text_config = vllm_config.model_config.hf_config.get_text_config() text_config = vllm_config.model_config.hf_config.get_text_config()
@@ -81,8 +85,10 @@ class WorkerLoRAManager:
def create_lora_manager( def create_lora_manager(
self, self,
model: torch.nn.Module, model: torch.nn.Module,
vllm_config: VllmConfig | None = None, vllm_config: VllmConfig,
) -> Any: ) -> Any:
if vllm_config is None:
raise ValueError("vllm_config must be provided to create a LoRA manager.")
lora_manager = create_lora_manager( lora_manager = create_lora_manager(
model, model,
max_num_seqs=self.max_num_seqs, max_num_seqs=self.max_num_seqs,
@@ -240,8 +246,10 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
def create_lora_manager( def create_lora_manager(
self, self,
model: torch.nn.Module, model: torch.nn.Module,
vllm_config: VllmConfig | None = None, vllm_config: VllmConfig,
) -> Any: ) -> Any:
if vllm_config is None:
raise ValueError("vllm_config must be provided to create a LoRA manager.")
lora_manager = create_lora_manager( lora_manager = create_lora_manager(
model, model,
lora_manager_cls=self._manager_cls, lora_manager_cls=self._manager_cls,
@@ -1029,6 +1029,7 @@ class FusedMoEKernelModularImpl:
): ):
self.prepare_finalize = prepare_finalize self.prepare_finalize = prepare_finalize
self.fused_experts = fused_experts self.fused_experts = fused_experts
self.shared_experts: SharedExperts | None = None
moe_parallel_config = fused_experts.moe_config.moe_parallel_config moe_parallel_config = fused_experts.moe_config.moe_parallel_config
self.moe_parallel_config = moe_parallel_config self.moe_parallel_config = moe_parallel_config
self.is_dp_ep = ( self.is_dp_ep = (
+2
View File
@@ -42,6 +42,7 @@ from .interfaces_base import VllmModel
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.lora.model_manager import LoRAModelManager
from vllm.model_executor.models.utils import WeightsMapper from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.multimodal.registry import _ProcessorFactories from vllm.multimodal.registry import _ProcessorFactories
@@ -554,6 +555,7 @@ class SupportsLoRA(Protocol):
packed_modules_mapping: dict[str, list[str]] = {} packed_modules_mapping: dict[str, list[str]] = {}
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers) # Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
lora_skip_prefixes: ClassVar[list[str]] = [] lora_skip_prefixes: ClassVar[list[str]] = []
lora_manager: "LoRAModelManager | None"
# We can't use runtime_checkable with ClassVar for issubclass checks # We can't use runtime_checkable with ClassVar for issubclass checks
+1 -3
View File
@@ -24,9 +24,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size, get_tensor_model_parallel_world_size,
) )
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import ( from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.logits_processor import LogitsProcessor