[MyPy] Fix mypy for vllm/lora (#41722)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-06-22 10:57:09 -04:00
committed by GitHub
parent 1c7bc18318
commit ccd49f6821
15 changed files with 140 additions and 50 deletions
+11 -2
View File
@@ -17,6 +17,7 @@ from vllm.forward_context import (
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
LinearBase,
QuantizeMethodBase,
ReplicatedLinear,
RowParallelLinear,
)
@@ -182,6 +183,14 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
lora_b, non_blocking=True
)
def _get_quant_method(self) -> QuantizeMethodBase:
quant_method = self.base_layer.quant_method
if quant_method is None:
raise RuntimeError(
f"{type(self.base_layer).__name__} must define quant_method for LoRA."
)
return quant_method
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
# is_forward_context_available for tower modules
if self._enable_aux_cuda_stream and is_forward_context_available():
@@ -195,7 +204,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
def _apply_sync(
self, x: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
output = self._get_quant_method().apply(self.base_layer, x, bias)
return self._apply_lora_to_output(x, output)
def _apply_base_forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -242,7 +251,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
output_size = sum(self.output_slices)
def base_fn() -> torch.Tensor:
return self.base_layer.quant_method.apply(self.base_layer, x, bias)
return self._get_quant_method().apply(self.base_layer, x, bias)
def lora_fn() -> torch.Tensor:
# Must be zeros, not empty: _lora_expand_kernel exits early (without
+14 -3
View File
@@ -33,7 +33,7 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
== len(layer.output_slices)
)
output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
output = layer._get_quant_method().apply(layer.base_layer, x, bias)
x = x.view(-1, x.shape[-1])
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
@@ -73,6 +73,8 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
)
if not current_platform.can_update_inplace():
if lora_output is None:
raise RuntimeError("LoRA expand must return an output tensor.")
output = lora_output
output = output.view(*out_orig_shape)
@@ -327,12 +329,16 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear)
base_forward = getattr(type(self.base_layer), "forward", None)
merged_forward = getattr(merged_cls, "forward", None)
# Effectively unsharded subclasses can safely reuse their custom
# forward() implementation before applying the LoRA delta.
if (
self.tp_size == 1
and type(self.base_layer) is not merged_cls
and type(self.base_layer).forward is not merged_cls.forward
and base_forward is not None
and merged_forward is not None
and base_forward is not merged_forward
):
return self._apply_base_forward(x)
return _mcp_apply(x, bias, self)
@@ -482,6 +488,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
lora_config: LoRAConfig,
packed_modules_list: list,
model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool:
return (
type(source_layer) is maybe_get_oot_by_class(QKVParallelLinear)
@@ -523,6 +530,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
lora_config: LoRAConfig,
packed_modules_list: list,
model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool:
# specifying kwargs so they can be easily accessed in decorator
return super().can_replace_layer(
@@ -565,6 +573,7 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo
lora_config: LoRAConfig,
packed_modules_list: list,
model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool:
# specifying kwargs so they can be easily accessed in decorator
return super().can_replace_layer(
@@ -650,6 +659,7 @@ class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
lora_config: LoRAConfig,
packed_modules_list: list,
model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool:
# specifying kwargs so they can be easily accessed in decorator
return super().can_replace_layer(
@@ -678,6 +688,7 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
lora_config: LoRAConfig,
packed_modules_list: list,
model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool:
# Support MergedColumnParallelLinear with 3 or more slices
# (2 slices are handled by MergedColumnParallelLinearWithLoRA)
@@ -727,7 +738,7 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
start_idx = 0
for output_size in output_sizes:
end_idx = start_idx + output_size
lora_b_list.append(lora_b[start_idx:end_idx, :])
lora_b_list.append(lora_b[start_idx:end_idx])
start_idx = end_idx
lora_b = lora_b_list
+25 -4
View File
@@ -12,10 +12,16 @@ from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.custom_op import maybe_get_oot_by_class
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
LoRAExpertsMixin,
)
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEKernel,
FusedMoEKernelModularImpl,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
@@ -58,6 +64,13 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
routed_experts._ensure_moe_quant_config_init()
if getattr(routed_experts.quant_method, "supports_internal_mk", False):
moe_kernel = routed_experts.quant_method.moe_kernel
assert moe_kernel is not None, (
"Fused MoE quant method must provide a moe_kernel."
)
# Don't let the kernel own shared experts so the runner can
# overlap them with routed experts via a separate CUDA stream.
assert isinstance(moe_kernel.impl, FusedMoEKernelModularImpl)
moe_kernel.impl.shared_experts = None
else:
prepare_finalize = MoEPrepareAndFinalizeNoDPEPModular()
moe_kernel = FusedMoEKernel(
@@ -405,7 +418,11 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
def set_mapping(self, punica_wrapper):
super().set_mapping(punica_wrapper)
lora_context = self._build_lora_context()
self._moe_kernel.fused_experts.set_lora_context(lora_context)
fused_experts = self._moe_kernel.fused_experts
assert isinstance(fused_experts, LoRAExpertsMixin), (
f"{type(fused_experts).__name__} does not support LoRA context setup."
)
fused_experts.set_lora_context(lora_context)
prepare_finalize = self._moe_kernel.prepare_finalize
if hasattr(prepare_finalize, "set_lora_context"):
prepare_finalize.set_lora_context(lora_context)
@@ -482,9 +499,13 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
) -> None:
"""Initializes lora matrices."""
assert isinstance(model_config, PretrainedConfig)
if model_config is None:
raise ValueError("model_config must be provided for MoE LoRA.")
architectures = model_config.architectures
if not architectures:
raise ValueError("model_config.architectures must be defined for MoE LoRA.")
self._verify_ep_fs(lora_config)
self._base_model = model_config.architectures[0]
self._base_model = architectures[0]
self.max_loras = lora_config.max_loras
self.fully_sharded = lora_config.fully_sharded_loras
+1 -1
View File
@@ -116,7 +116,7 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
return lora_b
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
output = self._get_quant_method().apply(self.base_layer, x, bias)
x = x.view(-1, x.shape[-1])
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
+11 -3
View File
@@ -111,19 +111,27 @@ def try_get_optimal_moe_lora_config(
# base MoE weight's block-wise quantization, so block_shape is omitted
# from the config lookup — the non-quantized branch in get_default_config
# ignores it anyway.
config = try_get_optimal_moe_config(w1_shape, w2_shape, top_k, dtype, M).copy()
raw_config = try_get_optimal_moe_config(w1_shape, w2_shape, top_k, dtype, M)
config: dict[str, int | None] = dict(raw_config)
if op_type in [
"fused_moe_lora_w13_shrink",
"fused_moe_lora_w2_shrink",
]:
block_size_n = config.get("BLOCK_SIZE_N")
config["BLOCK_SIZE_N"] = min(
config.get("BLOCK_SIZE_N", 64), next_power_of_2(rank)
block_size_n if block_size_n is not None else 64,
next_power_of_2(rank),
)
elif op_type in [
"fused_moe_lora_w13_expand",
"fused_moe_lora_w2_expand",
]:
block_size_k = config.get("BLOCK_SIZE_K")
config["BLOCK_SIZE_K"] = max(
16, min(config.get("BLOCK_SIZE_K", 32), next_power_of_2(rank))
16,
min(
block_size_k if block_size_k is not None else 32,
next_power_of_2(rank),
),
)
return config