[MyPy] Fix mypy for vllm/lora (#41722)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.forward_context import (
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearBase,
|
||||
QuantizeMethodBase,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
@@ -182,6 +183,14 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
|
||||
lora_b, non_blocking=True
|
||||
)
|
||||
|
||||
def _get_quant_method(self) -> QuantizeMethodBase:
|
||||
quant_method = self.base_layer.quant_method
|
||||
if quant_method is None:
|
||||
raise RuntimeError(
|
||||
f"{type(self.base_layer).__name__} must define quant_method for LoRA."
|
||||
)
|
||||
return quant_method
|
||||
|
||||
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
|
||||
# is_forward_context_available for tower modules
|
||||
if self._enable_aux_cuda_stream and is_forward_context_available():
|
||||
@@ -195,7 +204,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
|
||||
def _apply_sync(
|
||||
self, x: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
|
||||
output = self._get_quant_method().apply(self.base_layer, x, bias)
|
||||
return self._apply_lora_to_output(x, output)
|
||||
|
||||
def _apply_base_forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -242,7 +251,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
|
||||
output_size = sum(self.output_slices)
|
||||
|
||||
def base_fn() -> torch.Tensor:
|
||||
return self.base_layer.quant_method.apply(self.base_layer, x, bias)
|
||||
return self._get_quant_method().apply(self.base_layer, x, bias)
|
||||
|
||||
def lora_fn() -> torch.Tensor:
|
||||
# Must be zeros, not empty: _lora_expand_kernel exits early (without
|
||||
|
||||
@@ -33,7 +33,7 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
|
||||
== len(layer.output_slices)
|
||||
)
|
||||
|
||||
output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
|
||||
output = layer._get_quant_method().apply(layer.base_layer, x, bias)
|
||||
|
||||
x = x.view(-1, x.shape[-1])
|
||||
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
|
||||
@@ -73,6 +73,8 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
|
||||
)
|
||||
|
||||
if not current_platform.can_update_inplace():
|
||||
if lora_output is None:
|
||||
raise RuntimeError("LoRA expand must return an output tensor.")
|
||||
output = lora_output
|
||||
|
||||
output = output.view(*out_orig_shape)
|
||||
@@ -327,12 +329,16 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
|
||||
merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear)
|
||||
base_forward = getattr(type(self.base_layer), "forward", None)
|
||||
merged_forward = getattr(merged_cls, "forward", None)
|
||||
# Effectively unsharded subclasses can safely reuse their custom
|
||||
# forward() implementation before applying the LoRA delta.
|
||||
if (
|
||||
self.tp_size == 1
|
||||
and type(self.base_layer) is not merged_cls
|
||||
and type(self.base_layer).forward is not merged_cls.forward
|
||||
and base_forward is not None
|
||||
and merged_forward is not None
|
||||
and base_forward is not merged_forward
|
||||
):
|
||||
return self._apply_base_forward(x)
|
||||
return _mcp_apply(x, bias, self)
|
||||
@@ -482,6 +488,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
lora_config: LoRAConfig,
|
||||
packed_modules_list: list,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
decorate: bool = True,
|
||||
) -> bool:
|
||||
return (
|
||||
type(source_layer) is maybe_get_oot_by_class(QKVParallelLinear)
|
||||
@@ -523,6 +530,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
|
||||
lora_config: LoRAConfig,
|
||||
packed_modules_list: list,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
decorate: bool = True,
|
||||
) -> bool:
|
||||
# specifying kwargs so they can be easily accessed in decorator
|
||||
return super().can_replace_layer(
|
||||
@@ -565,6 +573,7 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo
|
||||
lora_config: LoRAConfig,
|
||||
packed_modules_list: list,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
decorate: bool = True,
|
||||
) -> bool:
|
||||
# specifying kwargs so they can be easily accessed in decorator
|
||||
return super().can_replace_layer(
|
||||
@@ -650,6 +659,7 @@ class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
|
||||
lora_config: LoRAConfig,
|
||||
packed_modules_list: list,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
decorate: bool = True,
|
||||
) -> bool:
|
||||
# specifying kwargs so they can be easily accessed in decorator
|
||||
return super().can_replace_layer(
|
||||
@@ -678,6 +688,7 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
|
||||
lora_config: LoRAConfig,
|
||||
packed_modules_list: list,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
decorate: bool = True,
|
||||
) -> bool:
|
||||
# Support MergedColumnParallelLinear with 3 or more slices
|
||||
# (2 slices are handled by MergedColumnParallelLinearWithLoRA)
|
||||
@@ -727,7 +738,7 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
|
||||
start_idx = 0
|
||||
for output_size in output_sizes:
|
||||
end_idx = start_idx + output_size
|
||||
lora_b_list.append(lora_b[start_idx:end_idx, :])
|
||||
lora_b_list.append(lora_b[start_idx:end_idx])
|
||||
start_idx = end_idx
|
||||
lora_b = lora_b_list
|
||||
|
||||
|
||||
@@ -12,10 +12,16 @@ from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.custom_op import maybe_get_oot_by_class
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
|
||||
LoRAExpertsMixin,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
|
||||
FusedMoEModularMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEKernel,
|
||||
FusedMoEKernelModularImpl,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
@@ -58,6 +64,13 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
routed_experts._ensure_moe_quant_config_init()
|
||||
if getattr(routed_experts.quant_method, "supports_internal_mk", False):
|
||||
moe_kernel = routed_experts.quant_method.moe_kernel
|
||||
assert moe_kernel is not None, (
|
||||
"Fused MoE quant method must provide a moe_kernel."
|
||||
)
|
||||
# Don't let the kernel own shared experts so the runner can
|
||||
# overlap them with routed experts via a separate CUDA stream.
|
||||
assert isinstance(moe_kernel.impl, FusedMoEKernelModularImpl)
|
||||
moe_kernel.impl.shared_experts = None
|
||||
else:
|
||||
prepare_finalize = MoEPrepareAndFinalizeNoDPEPModular()
|
||||
moe_kernel = FusedMoEKernel(
|
||||
@@ -405,7 +418,11 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
def set_mapping(self, punica_wrapper):
|
||||
super().set_mapping(punica_wrapper)
|
||||
lora_context = self._build_lora_context()
|
||||
self._moe_kernel.fused_experts.set_lora_context(lora_context)
|
||||
fused_experts = self._moe_kernel.fused_experts
|
||||
assert isinstance(fused_experts, LoRAExpertsMixin), (
|
||||
f"{type(fused_experts).__name__} does not support LoRA context setup."
|
||||
)
|
||||
fused_experts.set_lora_context(lora_context)
|
||||
prepare_finalize = self._moe_kernel.prepare_finalize
|
||||
if hasattr(prepare_finalize, "set_lora_context"):
|
||||
prepare_finalize.set_lora_context(lora_context)
|
||||
@@ -482,9 +499,13 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
|
||||
) -> None:
|
||||
"""Initializes lora matrices."""
|
||||
|
||||
assert isinstance(model_config, PretrainedConfig)
|
||||
if model_config is None:
|
||||
raise ValueError("model_config must be provided for MoE LoRA.")
|
||||
architectures = model_config.architectures
|
||||
if not architectures:
|
||||
raise ValueError("model_config.architectures must be defined for MoE LoRA.")
|
||||
self._verify_ep_fs(lora_config)
|
||||
self._base_model = model_config.architectures[0]
|
||||
self._base_model = architectures[0]
|
||||
self.max_loras = lora_config.max_loras
|
||||
self.fully_sharded = lora_config.fully_sharded_loras
|
||||
|
||||
|
||||
@@ -116,7 +116,7 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
|
||||
return lora_b
|
||||
|
||||
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
|
||||
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
|
||||
output = self._get_quant_method().apply(self.base_layer, x, bias)
|
||||
|
||||
x = x.view(-1, x.shape[-1])
|
||||
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
|
||||
|
||||
@@ -111,19 +111,27 @@ def try_get_optimal_moe_lora_config(
|
||||
# base MoE weight's block-wise quantization, so block_shape is omitted
|
||||
# from the config lookup — the non-quantized branch in get_default_config
|
||||
# ignores it anyway.
|
||||
config = try_get_optimal_moe_config(w1_shape, w2_shape, top_k, dtype, M).copy()
|
||||
raw_config = try_get_optimal_moe_config(w1_shape, w2_shape, top_k, dtype, M)
|
||||
config: dict[str, int | None] = dict(raw_config)
|
||||
if op_type in [
|
||||
"fused_moe_lora_w13_shrink",
|
||||
"fused_moe_lora_w2_shrink",
|
||||
]:
|
||||
block_size_n = config.get("BLOCK_SIZE_N")
|
||||
config["BLOCK_SIZE_N"] = min(
|
||||
config.get("BLOCK_SIZE_N", 64), next_power_of_2(rank)
|
||||
block_size_n if block_size_n is not None else 64,
|
||||
next_power_of_2(rank),
|
||||
)
|
||||
elif op_type in [
|
||||
"fused_moe_lora_w13_expand",
|
||||
"fused_moe_lora_w2_expand",
|
||||
]:
|
||||
block_size_k = config.get("BLOCK_SIZE_K")
|
||||
config["BLOCK_SIZE_K"] = max(
|
||||
16, min(config.get("BLOCK_SIZE_K", 32), next_power_of_2(rank))
|
||||
16,
|
||||
min(
|
||||
block_size_k if block_size_k is not None else 32,
|
||||
next_power_of_2(rank),
|
||||
),
|
||||
)
|
||||
return config
|
||||
|
||||
Reference in New Issue
Block a user