Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+1 -2
View File
@@ -244,8 +244,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
def _apply_async_impl(
self, x: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
"""
Forward pass with base linear and LoRA on separate CUDA streams
"""Forward pass with base linear and LoRA on separate CUDA streams
for overlap, using maybe_execute_in_parallel.
Base layer runs on default stream; LoRA runs on aux stream.
"""
+12 -19
View File
@@ -83,8 +83,7 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
"""
LoRA on top of ColumnParallelLinear layer.
"""LoRA on top of ColumnParallelLinear layer.
LoRA B is sliced for tensor parallelism.
There are two types for the `base_layer`:
1. ColumnParallelLinear, e.g.`dense_h_to_4h` in `FalconForCausalLM`.
@@ -140,6 +139,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
Returns:
- output
- bias
"""
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
@@ -210,8 +210,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
lora_config: LoRAConfig,
model_config: PretrainedConfig | None = None,
) -> None:
"""
The main reason for overriding this function is to enhance code
"""The main reason for overriding this function is to enhance code
maintainability.
"""
self.lora_config = lora_config
@@ -268,8 +267,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
lora_a: list[torch.Tensor | None],
lora_b: list[torch.Tensor | None],
) -> tuple[list[torch.Tensor | None], list[torch.Tensor | None]]:
"""
Expand packed adapter groups when they don't match n_slices.
"""Expand packed adapter groups when they don't match n_slices.
E.g. in_proj_qkv (covers Q+K+V) + in_proj_z.
A None group member means that member was not adapted; the slice(s)
@@ -388,8 +386,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
"""
ColumnParallelLinear layer that is specifically designed for
"""ColumnParallelLinear layer that is specifically designed for
qkv_proj. Certain models, such as chatglm3 and baichuan-7b,
only contains a single LoRA within their qkv_proj layer.
@@ -493,8 +490,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
lora_config: LoRAConfig,
model_config: PretrainedConfig | None = None,
) -> None:
"""
The main reason for overloading this function is to handle inconsistent
"""The main reason for overloading this function is to handle inconsistent
weight dimensions in qkv lora.
"""
super().create_lora_weights(max_loras, lora_config, model_config)
@@ -521,8 +517,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
"""
Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
"""Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
Based on S-LoRA, slicing happens along the rank dim.
"""
@@ -562,8 +557,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLoRA):
"""
Differs from MergedColumnParallelLinearWithLoRA by slicing the
"""Differs from MergedColumnParallelLinearWithLoRA by slicing the
LoRA A's also.
Based on S-LoRA, slicing happens along the rank dim.
@@ -605,8 +599,7 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo
class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
"""
Differs from QKVParallelLinearWithLoRA by slicing the
"""Differs from QKVParallelLinearWithLoRA by slicing the
LoRA A's also.
Based on S-LoRA, slicing happens along the rank dim.
@@ -641,8 +634,7 @@ class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
"""
Differs from MergedQKVParallelLinearWithLoRA by slicing the
"""Differs from MergedQKVParallelLinearWithLoRA by slicing the
LoRA A's also.
Based on S-LoRA, slicing happens along the rank dim.
@@ -741,7 +733,8 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
lora_b: torch.Tensor | list[torch.Tensor],
):
"""Override to handle single tensor weights
that need to be split into slices."""
that need to be split into slices.
"""
self.reset_lora(index)
# Handle case where checkpoint has single tensor weights
+7 -24
View File
@@ -248,7 +248,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
model_config: PretrainedConfig | None = None,
) -> None:
"""Initializes lora matrices."""
self._verify_ep_fs(lora_config)
self.max_loras = lora_config.max_loras
self.fully_sharded = lora_config.fully_sharded_loras
@@ -294,9 +293,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
)
def _slice_w13_a(self, w13_lora_a: torch.Tensor) -> torch.Tensor:
"""
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
"""
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
if self.tp_size == 1 or not self.fully_sharded:
return w13_lora_a
@@ -321,9 +318,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
return w13_lora_b[:, start_idx:end_idx, :]
def _slice_w2_a(self, w2_lora_a: torch.Tensor) -> torch.Tensor:
"""
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
"""
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
if self.tp_size == 1:
return w2_lora_a
# w2_lora_a shape (num_experts,rank,input_size)
@@ -334,9 +329,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
return w2_lora_a[:, :, start_idx:end_idx]
def _slice_w2_b(self, w2_lora_b: torch.Tensor) -> torch.Tensor:
"""
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
"""
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
if self.tp_size == 1 or not self.fully_sharded:
return w2_lora_b
# Based on S-LoRA, we slice W2 B along the hidden_size dim.
@@ -475,7 +468,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
model_config: PretrainedConfig | None = None,
) -> bool:
"""Returns True if the layer can be replaced by this LoRA layer."""
# source_layer is MoERunner
moe_cls = maybe_get_oot_by_class(MoERunner)
return isinstance(source_layer, moe_cls) and len(packed_modules_list) == 2
@@ -522,7 +514,6 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
model_config: PretrainedConfig | None = None,
) -> None:
"""Initializes lora matrices."""
if model_config is None:
raise ValueError("model_config must be provided for MoE LoRA.")
architectures = model_config.architectures
@@ -609,30 +600,22 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
@property
def w13_input_size(self):
"""
Full size
"""
"""Full size."""
return self.w13_lora_a_stacked[0].shape[-1]
@property
def w13_output_size(self):
"""
Full size
"""
"""Full size."""
return self.w13_lora_b_stacked[0].shape[-2] * self.tp_size
@property
def w2_input_size(self):
"""
Full size
"""
"""Full size."""
return self.w2_lora_a_stacked[0].shape[-1] * self.tp_size
@property
def w2_output_size(self):
"""
Full size
"""
"""Full size."""
return self.hidden_size
@classmethod
+2 -2
View File
@@ -18,8 +18,7 @@ from .base import BaseLayerWithLoRA
class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
"""
LoRA wrapper for LogitsProcessor, with extra logic to handle the
"""LoRA wrapper for LogitsProcessor, with extra logic to handle the
application of the LoRA adapter.
Args:
@@ -30,6 +29,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
sharded_to_full_mapping: index mapping from sharded vocab to full vocab
received from base_layer.get_sharded_to_full_mapping(). If None,
no reindexing will be done.
"""
def __init__(
+2 -1
View File
@@ -25,7 +25,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
def forward(
self, input_: torch.Tensor
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
"""Forward of ReplicatedLinearWithLoRA
"""Forward of ReplicatedLinearWithLoRA.
Args:
input_: Tensor whose last dimension is `input_size`.
@@ -33,6 +33,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
Returns:
- output
- bias
"""
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
+3 -3
View File
@@ -42,7 +42,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
def forward(
self, input_: torch.Tensor
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
"""Forward of RowParallelLinear
"""Forward of RowParallelLinear.
Args:
input_: tensor whose last dimension is `input_size`. If
@@ -52,6 +52,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
Returns:
- output
- bias
"""
# set up backprop all-reduce.
if self.base_layer.input_is_parallel:
@@ -99,8 +100,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
"""
Differs from RowParallelLinearWithLoRA by slicing the
"""Differs from RowParallelLinearWithLoRA by slicing the
LoRA B's also.
Based on S-LoRA, slicing happens along the output dim.
+2 -4
View File
@@ -74,8 +74,7 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
def _not_fully_sharded_can_replace(can_replace):
"""
decorator which adds the condition of not using fully sharded loras
"""Decorator which adds the condition of not using fully sharded loras
intended to wrap can_replace_layer()
"""
@@ -88,8 +87,7 @@ def _not_fully_sharded_can_replace(can_replace):
def _fully_sharded_can_replace(can_replace):
"""
decorator which adds the condition of fully sharded loras
"""Decorator which adds the condition of fully sharded loras
intended to wrap can_replace_layer()
"""
+20 -15
View File
@@ -39,9 +39,7 @@ _EXPERTS_SEPARATOR = ".experts."
def _is_remote_expert_key(raw_name: str, spec: "MoEEPLoadSpec") -> bool:
"""
Decide whether a checkpoint key belongs to a non-local expert.
"""
"""Decide whether a checkpoint key belongs to a non-local expert."""
pos = raw_name.find(_EXPERTS_SEPARATOR)
if pos < 0:
return False
@@ -69,15 +67,14 @@ class LoRAModel:
*,
modules_to_save: dict[str, LoRAFullModuleWeights] | None = None,
) -> None:
"""
Args:
lora_model_id: The integer id for the lora model.
rank: lora rank.
loras: module name -> weights for lora-replaced layers.
is_3d_lora_weight: Whether the on-disk MoE adapter is in the 3D
fused (gate_up_proj / down_proj) layout. Propagated from the
originating LoRARequest. Only consulted by the LoRA model
manager when enable_mixed_moe_lora_format is on.
"""Args:
lora_model_id: The integer id for the lora model.
rank: lora rank.
loras: module name -> weights for lora-replaced layers.
is_3d_lora_weight: Whether the on-disk MoE adapter is in the 3D
fused (gate_up_proj / down_proj) layout. Propagated from the
originating LoRARequest. Only consulted by the LoRA model
manager when enable_mixed_moe_lora_format is on.
"""
self.id = lora_model_id
@@ -93,7 +90,8 @@ class LoRAModel:
def clone(self, lora_model_id: int) -> "LoRAModel":
"""Return a copy of the object with different ids.
Will share the underlying tensors."""
Will share the underlying tensors.
"""
return self.__class__(
lora_model_id,
rank=self.rank,
@@ -103,7 +101,7 @@ class LoRAModel:
)
def get_lora(self, module_name: str) -> LoRALayerWeights | None:
"""Get LoRA for a given module by name"""
"""Get LoRA for a given module by name."""
return self.loras.get(module_name, None)
def get_module_to_save(self, module_name: str) -> LoRAFullModuleWeights | None:
@@ -114,7 +112,7 @@ class LoRAModel:
@staticmethod
def _should_skip_module(module_name: str, skip_prefixes: list[str]) -> bool:
"""Check if a module should be skipped based on skip prefixes"""
"""Check if a module should be skipped based on skip prefixes."""
for prefix in skip_prefixes:
if f".{prefix}" in module_name or module_name.startswith(prefix):
return True
@@ -222,6 +220,12 @@ class LoRAModel:
a global counter.
device: Device where the lora model is loaded.
dtype: dtype of the lora model weights.
model_vocab_size: Vocab size of the base model, used to size the
embedding deltas.
weights_mapper: Optional mapper rewriting checkpoint weight names
to vLLM names.
tensorizer_config_dict: Optional tensorizer config used to load
the checkpoint via tensorizer instead of from disk.
skip_prefixes: List of module name prefixes to skip during loading.
Models can define this to skip modules not used in inference
(e.g., MTP layers). Format: ["mtp."]
@@ -233,6 +237,7 @@ class LoRAModel:
Returns:
Loaded LoRA Model.
"""
lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
lora_bin_file_path = os.path.join(lora_dir, "adapter_model.bin")
-1
View File
@@ -163,7 +163,6 @@ class PackedLoRALayerWeights(LoRALayerWeights):
If LoRA is None, it signifies that the submodule does not have a LoRA.
"""
first_lora = next(lora for lora in loras if lora is not None)
assert first_lora is not None
rank = first_lora.rank
+6 -6
View File
@@ -97,6 +97,9 @@ class LoRAModelManager:
in a single batch.
vocab_size: the vocab size of the model.
lora_config: the LoRA configuration.
device: the device the LoRA tensors are placed on.
vllm_config: the vLLM config for this engine.
"""
self.model: SupportsLoRAModel = model
self.supported_lora_modules = get_supported_lora_modules(self.model)
@@ -785,6 +788,7 @@ class LoRAModelManager:
Returns:
True if the module passes the filter, False otherwise.
"""
return is_in_target_modules(
module_name,
@@ -793,9 +797,7 @@ class LoRAModelManager:
)
def _get_punica_wrapper(self, module_name: str) -> PunicaWrapperBase | None:
"""
Determine whether this module supports LoRA and which wrapper to use.
"""
"""Determine whether this module supports LoRA and which wrapper to use."""
# For language model (early return)
if not self.supports_mm:
return self.punica_wrapper_mapping[DEFAULT_LANGUAGE_WRAPPER_KEY]
@@ -1190,9 +1192,7 @@ class LoRAModelManager:
return new_module_names[start:end]
def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None:
"""
Per-rank slicing metadata for 2D RoutedEXperts LoRA modules.
"""
"""Per-rank slicing metadata for 2D RoutedEXperts LoRA modules."""
if not self._use_ep or not self._is_moe:
return None
module = next(
+7 -14
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Utilities for Punica kernel construction.
"""
"""Utilities for Punica kernel construction."""
from vllm.triton_utils import tl, triton
@@ -21,8 +19,7 @@ def _accumulate_mm(
group_n: tl.constexpr,
use_fp8_w8a8: tl.constexpr,
):
"""
Core matrix multiplication and accumulation logic with quantization support.
"""Core matrix multiplication and accumulation logic with quantization support.
Args:
tiled_a (tl.tensor): Loaded tile from A matrix
@@ -36,8 +33,8 @@ def _accumulate_mm(
group_k: Block size for K dimension in block-wise quantization
group_n: Block size for N dimension in block-wise quantization
use_fp8_w8a8: Whether using FP8 W8A8 quantization
"""
"""
if use_fp8_w8a8:
if group_k > 0 and group_n > 0:
# Block-wise quantization: scales are loaded per block
@@ -85,8 +82,7 @@ def fp8_mm_k(
USE_GDC: tl.constexpr,
base_k,
):
"""
FP8-compatible matrix multiplication kernel with quantization support.
"""FP8-compatible matrix multiplication kernel with quantization support.
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
B (k x n), iterate through the K dimension to compute the partial/complete
matrix block product with proper dequantization.
@@ -120,6 +116,7 @@ def fp8_mm_k(
b_dtype: datatype of the B matrix
USE_GDC: Whether to use PDL. True indicates use.
base_k (int): Base offset along K dimension for current SPLIT_K group
"""
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
@@ -238,13 +235,11 @@ def do_shrink_kernel_fp8(
per_channel_quant: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""
Given an array of integers that identifies the rows of A, ram,
"""Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice, compute the
matrix product and store in the appropriate output location.
"""
# Identify the lora_ptr from slice_id.
if SLICE_NUM == 1:
cur_lora_ptr = lora_ptr
@@ -432,8 +427,7 @@ def do_expand_kernel_fp8(
use_fp8_w8a8: tl.constexpr,
per_channel_quant: tl.constexpr,
):
"""
FP8-compatible expand kernel for LoRA.
"""FP8-compatible expand kernel for LoRA.
Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice,
@@ -449,7 +443,6 @@ def do_expand_kernel_fp8(
- Per-channel quantization
- Tensor-wise quantization
"""
# ls_d*_ptr can be either an integer or a pointer
if SAME_STRIDE:
cur_lora_d0_stride = ls_d0_ptr
@@ -24,7 +24,7 @@ def _get_lora_id(
top_k_num,
naive_block_assignment: tl.constexpr,
):
"""Returns lora_id"""
"""Returns lora_id."""
if naive_block_assignment:
token_idx = pid_m // top_k_num
return tl.load(token_lora_mapping_ptr + token_idx)
@@ -41,7 +41,7 @@ def _get_expert_id(
max_loras,
naive_block_assignment: tl.constexpr,
):
"""Returns expert_id"""
"""Returns expert_id."""
if naive_block_assignment:
return tl.load(expert_ids_ptr + pid_m)
else:
@@ -61,7 +61,7 @@ def _get_token_offs(
naive_block_assignment: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
):
"""Returns token offsets"""
"""Returns token offsets."""
if naive_block_assignment:
return tl.where(offs == 0, pid_m, num_valid_tokens)
else:
@@ -76,8 +76,7 @@ _LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
"""
`_LORA_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
@@ -101,9 +100,7 @@ def _adjust_kernel_inputs(
sorted_token_ids: torch.Tensor | None,
expert_ids: torch.Tensor,
):
"""
helper function to adjust kernel inputs when sorted_token_ids is None
"""
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
if sorted_token_ids is None:
stride_tl = 0
stride_el = 0
@@ -25,7 +25,7 @@ def _get_lora_id(
top_k_num,
naive_block_assignment: tl.constexpr,
):
"""Returns lora_id"""
"""Returns lora_id."""
if naive_block_assignment:
token_idx = pid_m // top_k_num
return tl.load(token_lora_mapping_ptr + token_idx)
@@ -42,7 +42,7 @@ def _get_expert_id(
max_loras,
naive_block_assignment: tl.constexpr,
):
"""Returns expert_id"""
"""Returns expert_id."""
if naive_block_assignment:
return tl.load(expert_ids_ptr + pid_m)
else:
@@ -62,7 +62,7 @@ def _get_token_offs(
naive_block_assignment: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
):
"""Returns token offsets"""
"""Returns token offsets."""
if naive_block_assignment:
return tl.where(offs == 0, pid_m, num_valid_tokens)
else:
@@ -852,8 +852,7 @@ def _run_fused_moe_lora_small_batch(
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
"""
`_LORA_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
@@ -877,9 +876,7 @@ def _adjust_kernel_inputs(
sorted_token_ids: torch.Tensor | None,
expert_ids: torch.Tensor,
):
"""
helper function to adjust kernel inputs when sorted_token_ids is None
"""
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
if sorted_token_ids is None:
stride_tl = 0
stride_el = 0
+15 -19
View File
@@ -1,19 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Utilities for Punica kernel construction.
"""
"""Utilities for Punica kernel construction."""
from vllm.triton_utils import tl, triton
@triton.jit
def mm_k(
a_ptr,
b_ptr,
ak_stride,
bk_stride,
offset_k,
a_ptr: tl.tensor,
b_ptr: tl.tensor,
ak_stride: int,
bk_stride: int,
offset_k: tl.tensor,
K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
@@ -23,32 +21,34 @@ def mm_k(
CAST_TYPE: tl.constexpr,
b_dtype: tl.constexpr,
USE_GDC: tl.constexpr,
base_k,
base_k: int,
):
"""
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
"""Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
B (k x n), iterate, through the K dimension to compute the partial/complete
matrix block product.
If SPLIT_K == 1, the output m x n product is complete.
If SPLIT_K > 1, the thread block computes partial outputs. The partial
outputs are then atomically summed in the caller code.
Args:
a_ptr: Array of pointers, identifying rows of A
b_ptr: Array of pointers, identifying columns of B
ak_stride: K dimension stride of the A matrix
bk_stride: K dimension stride of the B matrix
offset_k: Offsets within the current BLOCK_K tile along K
K: Length of the K dimension
BLOCK_M: M dimension of the output block m x n
BLOCK_N: N dimension of the output block m x n
BLOCK_K: K dimension atom
EVEN_K: True if the blocks of A and B can be loaded without any
masking.
masking.
SPLIT_K: Parameter signifying parallelism in the K dimension.
CAST_TYPE: if True, cast the values from the A matrix to the B
matrix dtype.
matrix dtype.
b_dtype: datatype of the B matrix
USE_GDC: Whether to use PDL. True indicates use.
base_k: Base offset along K dimension for current SPLIT_K group
"""
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
@@ -142,15 +142,13 @@ def do_expand_kernel(
ADD_INPUTS: tl.constexpr,
USE_GDC: tl.constexpr,
):
"""
Given an array of integers that identifies the rows of A, ram,
"""Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice,
compute the matrix product and store in the appropriate output location.
Given that this is an expand kernel, we don't perform any split-K reduction
as the K dimension is assumed to be small.
"""
# ls_d*_ptr can be either an integer or a pointer
if SAME_STRIDE:
# integer
@@ -268,13 +266,11 @@ def do_shrink_kernel(
SLICE_NUM: tl.constexpr,
USE_GDC: tl.constexpr,
):
"""
Given an array of integers that identifies the rows of A, ram,
"""Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice, compute the
matrix product and store in the appropriate output location.
"""
# Identify the lora_ptr from slice_id.
if SLICE_NUM == 1:
# current lora ptr
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -21,8 +20,7 @@ _EXPAND_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
def _get_expand_lora_scale_ptr(lora_weights: list[torch.Tensor], device: torch.device):
"""
`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
"""`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
`profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
@@ -91,9 +89,7 @@ def _lora_expand_kernel_fp8(
per_channel_quant: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""
FP8-compatible expand kernel wrapper.
"""
"""FP8-compatible expand kernel wrapper."""
cta_n_num = tl.cdiv(N, BLOCK_N)
cta_m_num = tl.cdiv(M, BLOCK_M)
@@ -192,8 +188,7 @@ def _lora_expand_fp8(
use_fp8_w8a8: bool = False,
per_channel_quant: bool = False,
) -> None:
"""
FP8-compatible LoRA expand operation.
"""FP8-compatible LoRA expand operation.
Args:
inputs: Input tensor from shrink operation [num_slices, num_tokens, lora_rank]
@@ -206,6 +201,8 @@ def _lora_expand_fp8(
num_tokens_per_lora: Number of tokens per LoRA
lora_token_start_loc: Start location for each LoRA's tokens
lora_ids: LoRA IDs to process
num_active_loras: Number of active LoRAs. Accepted for API parity with
the non-FP8 kernel and unused here.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
offset_start (int, optional): Offset start for output_tensor.
@@ -216,6 +213,7 @@ def _lora_expand_fp8(
group_n (int, optional): Block size for N in block-wise quantization.
use_fp8_w8a8 (bool, optional): Whether to use FP8 W8A8 quantization.
per_channel_quant (bool, optional): Whether to use per-channel quantization.
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
+25 -27
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -147,32 +146,31 @@ def _lora_expand(
offset_start: int = 0,
add_inputs: bool = False,
) -> None:
"""
Args:
inputs (torch.Tensor): input tensor
lora_b_weights (list[torch.Tensor]): lora'b weight
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
offset_start (int, optional): Offset start for output_tensor.
Defaults to 0.
add_inputs (bool, optional): Whether to add the input tensor to the
output tensor. Defaults to False.
"""
"""Args:
inputs (torch.Tensor): input tensor
lora_b_weights (list[torch.Tensor]): lora'b weight
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
offset_start (int, optional): Offset start for output_tensor.
Defaults to 0.
add_inputs (bool, optional): Whether to add the input tensor to the
output tensor. Defaults to False.
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
# None of the inputs require LoRA.
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
LoRA kernels metadata preparation utilities.
"""
"""LoRA kernels metadata preparation utilities."""
import bisect
from dataclasses import dataclass, field
@@ -107,14 +105,13 @@ class LoRAKernelMeta:
self.num_active_loras_cpu.fill_(0)
def prepare_tensors(self, token_lora_mapping: torch.Tensor) -> None:
"""
Prepare kernel metadata tensors for the current forward pass.
"""Prepare kernel metadata tensors for the current forward pass.
Args:
token_lora_mapping (torch.Tensor): Tensor containing lora indices
for each input token.
"""
"""
self._reset()
# Check and record no-lora case.
@@ -180,8 +177,7 @@ class LoRAKernelMeta:
torch.Tensor,
torch.Tensor,
]:
"""
This function returns the kernel metadata required for the current
"""This function returns the kernel metadata required for the current
forward pass execution of the kernel. The function returns all the
metadata required by the kernel, in order, as a tuple, so it can be
unpacked directly during the lora_shrink/lora_expand function call.
@@ -189,6 +185,10 @@ class LoRAKernelMeta:
Args:
token_nums (int): Number of input tokens in the current forward
pass of the kernel.
specialize_active_lora (bool): If True, report the actual active
LoRA count; otherwise report the padded default, which keeps
the kernel launch shape stable.
"""
if specialize_active_lora:
num_active_loras = self.num_active_loras_cpu
+19 -21
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -20,8 +19,7 @@ _SHRINK_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], tuple] = {}
def _get_shrink_lora_scale_ptr(
lora_scale_weights: list[torch.Tensor], device: torch.device
):
"""
`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
"""`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
`profile_run`. After this, it remains constant and subsequent usage is
through LUT.
@@ -241,23 +239,23 @@ def _lora_shrink_fp8(
use_fp8_w8a8: bool = False,
per_channel_quant: bool = False,
) -> None:
"""
Args:
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
output_tensor: Output tensor (FP16/BF16/FP32)
token_lora_mapping: Token to LoRA ID mapping
token_indices_sorted_by_lora_ids: Sorted token indices
num_tokens_per_lora: Number of tokens per LoRA
lora_token_start_loc: Start location for each LoRA's tokens
lora_ids: LoRA IDs to process
scaling: LoRA scaling factor
a_scale: Activation quantization scales
b_scale: Weight quantization scales per slice
group_k: Block size for K dimension quantization
group_n: Block size for N dimension quantization
use_fp8_w8a8: Whether to use FP8 weights and activations
per_channel_quant: Whether to use per-channel quantization
"""Args:
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
output_tensor: Output tensor (FP16/BF16/FP32)
token_lora_mapping: Token to LoRA ID mapping
token_indices_sorted_by_lora_ids: Sorted token indices
num_tokens_per_lora: Number of tokens per LoRA
lora_token_start_loc: Start location for each LoRA's tokens
lora_ids: LoRA IDs to process
scaling: LoRA scaling factor
a_scale: Activation quantization scales
b_scale: Weight quantization scales per slice
group_k: Block size for K dimension quantization
group_n: Block size for N dimension quantization
use_fp8_w8a8: Whether to use FP8 weights and activations
per_channel_quant: Whether to use per-channel quantization
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
+25 -27
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -142,32 +141,31 @@ def _lora_shrink(
num_active_loras: torch.Tensor, # CPU tensor [1], number of active LoRAs
scaling: float,
) -> None:
"""
Args:
inputs (torch.Tensor): Input tensor
lora_a_weights (list[torch.Tensor]): LoRA weights
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
number of active LoRAs. Stored as a tensor (not int) so
torch.compile treats it as dynamic rather than a constant.
scaling (float): Scaling factor.
"""
"""Args:
inputs (torch.Tensor): Input tensor
lora_a_weights (list[torch.Tensor]): LoRA weights
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
number of active LoRAs. Stored as a tensor (not int) so
torch.compile treats it as dynamic rather than a constant.
scaling (float): Scaling factor.
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
# None of the inputs require LoRA.
+3 -8
View File
@@ -23,8 +23,7 @@ _LORA_B_PTR_DICT: dict[tuple[int, ...], tuple[torch.tensor, ...]] = {}
def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
"""
`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
@@ -75,14 +74,12 @@ def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
def _get_lora_b_ptr(
lora_weights: list[torch.Tensor], offset_start: int, device: torch.device
):
"""
`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
"""
key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights)
if values := _LORA_B_PTR_DICT.get(key):
return values
@@ -317,9 +314,7 @@ def get_lora_op_configs(
@lru_cache
def supports_pdl(device: torch.device | None = None) -> bool:
"""
Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py
"""
"""Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py."""
# PDL requires compute capability SM90 or above
return (
+3 -7
View File
@@ -18,8 +18,7 @@ logger = init_logger(__name__)
@dataclass
class PEFTHelper:
"""
A helper class for PEFT configurations, specifically designed for LoRA.
"""A helper class for PEFT configurations, specifically designed for LoRA.
This class handles configuration validation, compatibility checks for
various LoRA implementations.
"""
@@ -40,9 +39,7 @@ class PEFTHelper:
vllm_max_position_embeddings: int | None = field(default=False)
def _validate_features(self) -> list[str]:
"""
Check if there are any unsupported LoRA features.
"""
"""Check if there are any unsupported LoRA features."""
error_msg = []
if self.modules_to_save:
unsupported_modules = [
@@ -124,8 +121,7 @@ class PEFTHelper:
return cls.from_dict(config)
def validate_legal(self, lora_config: LoRAConfig) -> None:
"""
Validates the LoRA configuration settings against application
"""Validates the LoRA configuration settings against application
constraints and requirements.
"""
error_msg = self._validate_features()
+23 -49
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -20,9 +19,7 @@ if TYPE_CHECKING:
class PunicaWrapperABC(ABC):
"""
PunicaWrapper ABC.
"""
"""PunicaWrapper ABC."""
@abstractmethod
def update_metadata(
@@ -33,9 +30,7 @@ class PunicaWrapperABC(ABC):
vocab_size: int,
**kwargs,
) -> None:
"""
Update the lora-related metadata
"""
"""Update the lora-related metadata"""
raise NotImplementedError
@abstractmethod
@@ -47,10 +42,7 @@ class PunicaWrapperABC(ABC):
scale: float,
**kwargs,
) -> torch.Tensor | None:
"""
Performs GEMM for multiple slices of lora_a.
"""
"""Performs GEMM for multiple slices of lora_a."""
raise NotImplementedError
@abstractmethod
@@ -64,9 +56,7 @@ class PunicaWrapperABC(ABC):
add_inputs=True,
**kwargs,
) -> torch.Tensor | None:
"""
Performs GEMM for multiple slices of lora_b.
"""
"""Performs GEMM for multiple slices of lora_b."""
raise NotImplementedError
@abstractmethod
@@ -78,8 +68,7 @@ class PunicaWrapperABC(ABC):
add_inputs: bool = True,
**kwargs,
) -> torch.Tensor | None:
"""
Applies lora specifically for VocabParallelEmbeddingWithLoRA,
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA,
and this layer only requires the expand operation.
"""
raise NotImplementedError
@@ -97,10 +86,7 @@ class PunicaWrapperABC(ABC):
buffer: tuple[torch.Tensor, ...] | None = None,
**kwargs,
) -> torch.Tensor | None:
"""
Applicable to linear-related lora.
"""
"""Applicable to linear-related lora."""
raise NotImplementedError
@abstractmethod
@@ -115,9 +101,7 @@ class PunicaWrapperABC(ABC):
buffer: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | None:
"""
Applies lora specifically for LogitsProcessorWithLoRA.
"""
"""Applies lora specifically for LogitsProcessorWithLoRA."""
raise NotImplementedError
@abstractmethod
@@ -134,8 +118,7 @@ class PunicaWrapperABC(ABC):
class PunicaWrapperBase(PunicaWrapperABC):
"""
PunicaWrapperBase is designed to manage and provide metadata for the punica
"""PunicaWrapperBase is designed to manage and provide metadata for the punica
kernel. The main function is to maintain the state information for
Multi-LoRA, and to provide the interface for the punica.
"""
@@ -249,8 +232,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
def prefill_metadata(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
"""
This property provides a convenient way to access the necessary
"""This property provides a convenient way to access the necessary
metadata for prefill-related kernel computations.
1. seq_start_locs: Tensor of sequence start positions.
2. seq_lengths: Tensor of sequence lengths.
@@ -271,8 +253,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
@property
def token_lora_indices(self) -> torch.Tensor:
"""
This property provides the lora indices corresponding to each token
"""This property provides the lora indices corresponding to each token
in the batch. An index of -1 means no lora should be applied.
"""
token_lora_len = self.indices_len[0]
@@ -280,8 +261,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
@property
def sampler_indices(self) -> torch.Tensor:
"""
This property is used to access the lora indices specifically for
"""This property is used to access the lora indices specifically for
LogitsProcessorWithLoRA.
"""
sampler_indices_len = self.indices_len[1]
@@ -289,9 +269,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
@property
def sampler_indices_padded(self) -> torch.Tensor:
"""
This property provides access to padded sampler indices.
"""
"""This property provides access to padded sampler indices."""
indices_padded_len = self.indices_len[2]
return self._sampler_indices_padded[:indices_padded_len]
@@ -321,8 +299,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
scale: float,
**kwargs,
) -> torch.Tensor | None:
"""
Performs GEMM for multiple slices of lora_a.
"""Performs GEMM for multiple slices of lora_a.
Semantics:
for i in range(len(lora_a_stacked)):
@@ -349,8 +326,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
add_inputs=True,
**kwargs,
) -> torch.Tensor | None:
"""
Performs GEMM for multiple slices of lora_b.
"""Performs GEMM for multiple slices of lora_b.
Semantics:
offset = offset_start
@@ -380,8 +356,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
add_inputs: bool = True,
**kwargs,
) -> torch.Tensor | None:
"""
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
and this layer only requires the expand operation.
Semantics:
y += x @ lora_b_stacked
@@ -391,6 +366,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
x (torch.Tensor): Input tensor.
lora_b_stacked (torch.Tensor): lora_b's weights.
add_inputs (bool): Default to True.
"""
# TODO: implement it based on torch ops
raise NotImplementedError
@@ -408,8 +384,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
buffer: tuple[torch.Tensor, ...] | None = None,
**kwargs,
) -> torch.Tensor | None:
"""
Applicable to linear-related lora.
"""Applicable to linear-related lora.
Semantics:
for i in range(len(lora_a_stacked)):
@@ -428,6 +403,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
scale (float): Scaling factor.
output_slices (tuple[int, ...]): Every slice's size.
buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None.
"""
# TODO: implement it based on torch ops
raise NotImplementedError
@@ -444,8 +420,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
buffer: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | None:
"""
Applies lora specifically for LogitsProcessorWithLoRA.
"""Applies lora specifically for LogitsProcessorWithLoRA.
Semantics:
buffer = (x @ lora_a_stacked) * scale
@@ -458,6 +433,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
lora_b_stacked (torch.Tensor):lora_b's weights.
scale (float): Scaling factor.
buffer (Optional[torch.Tensor]):Default to None.
"""
# TODO: implement it based on torch ops
raise NotImplementedError
@@ -474,8 +450,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
pad_sorted_ids: bool = False,
naive_block_assignment: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Aligns tokens and experts into block-sized chunks for LoRA-based
"""Aligns tokens and experts into block-sized chunks for LoRA-based
mixture-of-experts (MoE) execution.
"""
# TODO: implement it based on torch ops
@@ -501,8 +476,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
offset: int = 0,
token_lora_mapping: torch.Tensor | None = None,
):
"""
Performs a fused forward computation for LoRA of
"""Performs a fused forward computation for LoRA of
Mixture-of-Experts (MoE) layer.
"""
# TODO: implement it based on torch ops
+25 -25
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import Any
import torch
@@ -20,8 +21,7 @@ from .punica_base import PunicaWrapperBase
# The platforms that are compatible with the PyTorch-native implementation can
# inherit this class
class PunicaWrapperCPU(PunicaWrapperBase):
"""
PunicaWrapperCPU is designed to manage and provide metadata for the punica
"""PunicaWrapperCPU is designed to manage and provide metadata for the punica
kernel. The main function is to maintain the state information for
Multi-LoRA, and to provide the interface for the pytorch punica ops.
"""
@@ -133,12 +133,10 @@ class PunicaWrapperCPU(PunicaWrapperBase):
y_slice_size: int,
add_inputs: bool = True,
):
"""
Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
"""Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
computation, which is suitable for the
GEMM of lora'b.
"""
expand_slice_fun: Callable = (
self._expand_slice_prefill if self.is_prefill else self._expand_slice_decode
)
@@ -147,8 +145,7 @@ class PunicaWrapperCPU(PunicaWrapperBase):
def _apply_shrink(
self, y: torch.Tensor, x: torch.Tensor, w_t_all: torch.Tensor, scale: float
):
"""
Perform the ` y+=x@w_t_all` computation, which is suitable for the
"""Perform the ` y+=x@w_t_all` computation, which is suitable for the
GEMM of lora'a.
When `is_prefill is` true, it indicates that it is currently the
prefill stage, and the `_shrink_prefill` function should be called.
@@ -169,10 +166,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
scale: float,
**kwargs,
**kwargs: Any,
):
"""
Performs GEMM for multiple slices of lora_a.
"""Performs GEMM for multiple slices of lora_a.
When `is_prefill is` true, it indicates that it is currently the
prefill stage, and the `_shrink_prefill` function should be called.
Otherwise, it is the decode stage, and the _shrink_decode function
@@ -187,8 +183,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
x (torch.Tensor): Input tensor
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights
scale (float): Scaling factor for the operation
"""
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
x = x.view(-1, x.shape[-1])
# TODO fuse these kernels
for slice_idx in range(len(lora_a_stacked)):
@@ -202,10 +199,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
output_slices: tuple[int, ...],
offset_start: int = 0,
add_inputs=True,
**kwargs,
**kwargs: Any,
) -> None:
"""
Performs GEMM for multiple slices of lora_b.
"""Performs GEMM for multiple slices of lora_b.
Semantics:
for i in range(len(lora_b_stacked)):
@@ -218,7 +214,10 @@ class PunicaWrapperCPU(PunicaWrapperBase):
x (Union[tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
output_slices (tuple[int, ...]): Every slice's size
offset_start (int): The starting position of y, defaults to 0.
add_inputs (bool): Defaults to True.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
y_org = y
y = y.view(-1, y.shape[-1])
@@ -241,10 +240,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
x: torch.Tensor,
lora_b_stacked: torch.Tensor,
add_inputs: bool = True,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
Semantics:
y += x @ lora_b_stacked
@@ -254,8 +252,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
x (torch.Tensor): Input tensor.
lora_b_stacked (torch.Tensor): lora_b's weights.
add_inputs (bool): Default to True.
"""
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
# Embedding layer only need expand op
expand_fun: Callable = (
self._expand_prefill if self.is_prefill else self._expand_decode
@@ -272,10 +271,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
output_slices: tuple[int, ...],
*,
buffer: tuple[torch.Tensor, ...] | None = None,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applicable to linear-related lora.
"""Applicable to linear-related lora.
Semantics:
for i in range(len(lora_a_stacked)):
@@ -294,8 +292,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
scale (float): Scaling factor.
output_slices (tuple[int, ...]): Every slice's size.
buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None.
"""
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
if buffer is None:
@@ -320,10 +319,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
scale,
*,
buffer: torch.Tensor | None = None,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applies lora specifically for LogitsProcessorWithLoRA.
"""Applies lora specifically for LogitsProcessorWithLoRA.
Semantics:
buffer = (x @ lora_a_stacked) * scale
@@ -336,6 +334,8 @@ class PunicaWrapperCPU(PunicaWrapperBase):
lora_b_stacked (torch.Tensor):lora_b's weights.
scale (float): Scaling factor.
buffer (Optional[torch.Tensor]):Default to None.
**kwargs: Forwarded to `add_shrink` and `add_expand`.
"""
y_org = y
y = y.view(-1, y.shape[-1])
+28 -28
View File
@@ -1,13 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
"""
from typing import final
from typing import Any, final
import torch
@@ -32,8 +31,7 @@ from .punica_base import PunicaWrapperBase
@final
class PunicaWrapperGPU(PunicaWrapperBase):
"""
PunicaWrapperGPU is designed to manage and provide metadata for the punica
"""PunicaWrapperGPU is designed to manage and provide metadata for the punica
kernel. The main function is to maintain the state information for
Multi-LoRA, and to provide the interface for the punica triton kernel.
"""
@@ -96,10 +94,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
scale: float,
**kwargs,
**kwargs: Any,
):
"""
Performs GEMM for multiple slices of lora_a.
"""Performs GEMM for multiple slices of lora_a.
Semantics:
for i in range(len(lora_a_stacked)):
@@ -110,8 +107,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
x (torch.Tensor): Input tensor
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights
scale (float): Scaling factor for the operation
"""
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
x = x.view(-1, x.shape[-1])
lora_shrink(
x,
@@ -131,10 +129,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
output_slices: tuple[int, ...],
offset_start: int = 0,
add_inputs=True,
**kwargs,
**kwargs: Any,
) -> None:
"""
Performs GEMM for multiple slices of lora_b.
"""Performs GEMM for multiple slices of lora_b.
Semantics:
for i in range(len(lora_b_stacked)):
@@ -147,9 +144,12 @@ class PunicaWrapperGPU(PunicaWrapperBase):
x (torch.Tensor): Input tensors
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
output_slices (tuple[int, ...]): Every slice's size
offset_start (int): The starting position of y, defaults to 0.
add_inputs (bool): If True, add LoRA output to y; if False, write
LoRA-only output to y (used for dual-stream when base and LoRA
run on different CUDA streams). Defaults to True.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
y_org = y
y = y.view(-1, y.shape[-1])
@@ -177,10 +177,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
x: torch.Tensor,
lora_b_stacked: torch.Tensor,
add_inputs: bool = True,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
Semantics:
y += x @ lora_b_stacked
@@ -190,8 +189,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
x (torch.Tensor): Input tensor.
lora_b_stacked (torch.Tensor): lora_b's weights.
add_inputs (bool): Default to True.
"""
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
lora_expand(
x.unsqueeze(dim=0),
(lora_b_stacked,),
@@ -213,10 +213,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
output_slices: tuple[int, ...],
*,
buffer: torch.Tensor | None = None,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applicable to linear-related lora.
"""Applicable to linear-related lora.
Semantics:
for i in range(len(lora_a_stacked)):
@@ -226,6 +225,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
@ lora_b_stacked[indices[i], layer_idx, :, :]
* scale
).squeeze(0)
Args:
y (torch.Tensor): Output tensor. Will be changed in-place.
x (torch.Tensor): Input tensor
@@ -234,8 +234,10 @@ class PunicaWrapperGPU(PunicaWrapperBase):
scale (float): Scaling factor.
output_slices (tuple[int, ...]): Every slice's size.
buffer (Optional[torch.Tensor]): Defaults to None.
"""
**kwargs: Accepts `add_inputs` (bool, defaults to True); any other
keys are ignored.
"""
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
assert buffer is None, (
@@ -275,10 +277,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
scale,
*,
buffer: torch.Tensor | None = None,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applies lora specifically for LogitsProcessorWithLoRA.
"""Applies lora specifically for LogitsProcessorWithLoRA.
Semantics:
buffer = (x @ lora_a_stacked) * scale
@@ -291,6 +292,8 @@ class PunicaWrapperGPU(PunicaWrapperBase):
lora_b_stacked (torch.Tensor): lora_b's weights.
scale (float): Scaling factor.
buffer (Optional[torch.Tensor]): Default to None.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
y_org = y
y = y.view(-1, y.shape[-1])
@@ -367,8 +370,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
naive_block_assignment: bool = False,
token_lora_mapping: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Aligns tokens and experts into block-sized chunks for LoRA-based
"""Aligns tokens and experts into block-sized chunks for LoRA-based
mixture-of-experts (MoE) execution.
When `token_lora_mapping` is provided, it overrides the global mapping
@@ -467,9 +469,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
token_lora_mapping: torch.Tensor | None = None,
add_inputs: bool = True,
):
"""
Performs a fused forward computation for LoRA of Mixture-of-Experts (MoE) layer.
"""
"""Perform a fused forward computation for a LoRA MoE layer."""
(
token_lora_mapping_meta,
_,
+29 -30
View File
@@ -1,13 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
"""
from typing import final
from typing import Any, final
import torch
@@ -29,8 +28,7 @@ from .punica_base import PunicaWrapperBase
@final
class PunicaWrapperXPU(PunicaWrapperBase):
"""
PunicaWrapperXPU is designed to manage and provide metadata for the punica
"""PunicaWrapperXPU is designed to manage and provide metadata for the punica
kernel. The main function is to maintain the state information for
Multi-LoRA, and to provide the interface for the punica ipex kernel.
"""
@@ -135,10 +133,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
scale: float,
**kwargs,
**kwargs: Any,
):
"""
Performs GEMM for multiple slices of lora_a.
"""Performs GEMM for multiple slices of lora_a.
Semantics:
for i in range(len(lora_a_stacked)):
@@ -149,8 +146,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
x (torch.Tensor): Input tensor
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights
scale (float): Scaling factor for the operation
"""
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
x = x.view(-1, x.shape[-1])
for slice_idx in range(len(lora_a_stacked)):
self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx], scale)
@@ -163,10 +161,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
output_slices: tuple[int, ...],
offset_start: int = 0,
add_inputs=True,
**kwargs,
**kwargs: Any,
) -> None:
"""
Performs GEMM for multiple slices of lora_b.
"""Performs GEMM for multiple slices of lora_b.
Semantics:
for i in range(len(lora_b_stacked)):
@@ -179,7 +176,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
x (torch.Tensor): Input tensors
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
output_slices (tuple[int, ...]): Every slice's size
offset_start (int): The starting position of y, defaults to 0.
add_inputs (bool): Defaults to True.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
y_org = y
y = y.view(-1, y.shape[-1])
@@ -206,10 +206,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
x: torch.Tensor,
lora_b_stacked: torch.Tensor,
add_inputs: bool = True,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
Semantics:
y += x @ lora_b_stacked
@@ -219,6 +218,8 @@ class PunicaWrapperXPU(PunicaWrapperBase):
x (torch.Tensor): Input tensor.
lora_b_stacked (torch.Tensor): lora_b's weights.
add_inputs (bool): Default to True.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
token_lora_indices = self._get_token_lora_indices(x)
bgmv_expand(x, lora_b_stacked, y, token_lora_indices, add_inputs)
@@ -233,10 +234,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
output_slices: tuple[int, ...],
*,
buffer: torch.Tensor | None = None,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applicable to linear-related lora.
"""Applicable to linear-related lora.
Semantics:
for i in range(len(lora_a_stacked)):
@@ -246,6 +246,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
@ lora_b_stacked[indices[i], layer_idx, :, :]
* scale
).squeeze(0)
Args:
y (torch.Tensor): Output tensor. Will be changed in-place.
x (torch.Tensor): Input tensor
@@ -254,8 +255,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
scale (float): Scaling factor.
output_slices (tuple[int, ...]): Every slice's size.
buffer (Optional[torch.Tensor]): Defaults to None.
"""
**kwargs: Accepts `add_inputs` (bool, defaults to True); any other
keys are ignored.
"""
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
assert buffer is None, (
@@ -287,9 +290,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
@property
def sampler_indices_padded(self) -> torch.Tensor:
"""
This property provides access to padded sampler indices.
"""
"""This property provides access to padded sampler indices."""
return self._sampler_indices_padded[:]
def add_lora_logits(
@@ -301,10 +302,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
scale,
*,
buffer: torch.Tensor | None = None,
**kwargs,
**kwargs: Any,
) -> None:
"""
Applies lora specifically for LogitsProcessorWithLoRA.
"""Applies lora specifically for LogitsProcessorWithLoRA.
Semantics:
buffer = (x @ lora_a_stacked) * scale
@@ -317,6 +317,8 @@ class PunicaWrapperXPU(PunicaWrapperBase):
lora_b_stacked (torch.Tensor): lora_b's weights.
scale (float): Scaling factor.
buffer (Optional[torch.Tensor]): Default to None.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
y_org = y
y = y.view(-1, y.shape[-1])
@@ -366,8 +368,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
naive_block_assignment: bool = False,
token_lora_mapping: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Aligns tokens and experts into block-sized chunks for LoRA-based
"""Aligns tokens and experts into block-sized chunks for LoRA-based
mixture-of-experts (MoE) execution.
When `token_lora_mapping` is provided, it overrides the global mapping
@@ -466,9 +467,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
token_lora_mapping: torch.Tensor | None = None,
add_inputs: bool = True,
):
"""
Performs a fused forward computation for LoRA of Mixture-of-Experts (MoE) layer.
"""
"""Perform a fused forward computation for a LoRA MoE layer."""
(
token_lora_mapping_meta,
_,
+3 -3
View File
@@ -15,15 +15,13 @@ if TYPE_CHECKING:
def compute_meta(
token_lora_tensor: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
"""
Get the information required for the sgmv kernel. With the features:
"""Get the information required for the sgmv kernel. With the features:
1. If consecutive requests in the batch use the same LoRA, this function
will combine them into a single request, improving sgmv kernel inference
performance.
2. At the beginning of each prefill stage inference, recalculations are
needed based on the input, but only once.
"""
lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
token_lora_tensor, return_counts=True
)
@@ -65,6 +63,7 @@ def convert_mapping(
lora_index_to_id: List mapping LoRA ids to LoRA indices.
max_loras: Maximum number of LoRAs.
vocab_size: Model vocab size.
device: Device the returned index tensors are created on.
Returns:
A tuple of tensors:
@@ -83,6 +82,7 @@ def convert_mapping(
indices_len: List of lengths of the above tensors. It contains
(base_indices, sampler_indices, sampler_indices_padded,
embeddings_indices).
"""
index_mapping_indices: list[int] = list(mapping.index_mapping).copy()
embedding_indices = index_mapping_indices.copy()
+3 -6
View File
@@ -10,8 +10,7 @@ class LoRARequest(
omit_defaults=True, # type: ignore[call-arg]
array_like=True,
): # type: ignore[call-arg]
"""
Request for a LoRA adapter.
"""Request for a LoRA adapter.
lora_int_id must be globally unique for a given adapter.
This is currently not enforced in vLLM.
@@ -56,16 +55,14 @@ class LoRARequest(
return self.lora_path
def __eq__(self, value: object) -> bool:
"""
Overrides the equality method to compare LoRARequest
"""Overrides the equality method to compare LoRARequest
instances based on lora_name. This allows for identification
and comparison lora adapter across engines.
"""
return isinstance(value, self.__class__) and self.lora_name == value.lora_name
def __hash__(self) -> int:
"""
Overrides the hash method to hash LoRARequest instances
"""Overrides the hash method to hash LoRARequest instances
based on lora_name. This ensures that LoRARequest instances
can be used in hash-based collections such as sets and dictionaries,
identified by their names across engines.
+7
View File
@@ -36,6 +36,7 @@ class LoRAResolver(ABC):
Returns:
Optional[LoRARequest]: The resolved LoRA model information, or None
if the LoRA model cannot be found.
"""
pass
@@ -54,9 +55,11 @@ class _LoRAResolverRegistry:
resolver: LoRAResolver,
) -> None:
"""Register a LoRA resolver.
Args:
resolver_name: Name to register the resolver under.
resolver: The LoRA resolver instance to register.
"""
if resolver_name in self.resolvers:
logger.warning(
@@ -70,12 +73,16 @@ class _LoRAResolverRegistry:
def get_resolver(self, resolver_name: str) -> LoRAResolver:
"""Get a registered resolver instance by name.
Args:
resolver_name: Name of the resolver to get.
Returns:
The resolver instance.
Raises:
KeyError: If the resolver is not found in the registry.
"""
if resolver_name not in self.resolvers:
raise KeyError(
+12 -15
View File
@@ -49,8 +49,7 @@ logger = init_logger(__name__)
def get_captured_lora_counts(max_loras: int, specialize: bool) -> list[int]:
"""
Returns num_active_loras values for cudagraph capture.
"""Returns num_active_loras values for cudagraph capture.
When specialize=True: powers of 2 up to max_loras, plus max_loras + 1.
When specialize=False: just [max_loras + 1].
@@ -170,7 +169,7 @@ def parse_fine_tuned_lora_name(
) -> tuple[str, bool]:
"""Parse the name of lora weights.
args:
Args:
name: the name of the fine-tuned LoRA, e.g.
base_model.model.dense1.weight
weights_mapper: maps the name of weight, e.g.
@@ -179,8 +178,8 @@ def parse_fine_tuned_lora_name(
tuple(module_name, is_lora_a):
module_name: the name of the module, e.g. model.dense1,
is_lora_a whether the tensor is lora_a or lora_b.
"""
"""
# LoRA weight qualified name usually starts with `base_model.model.`,
# so we remove the prefix `base_model.model.` to make the following
# mapping correctly.
@@ -231,10 +230,7 @@ def is_base_embedding_weights(name: str) -> bool:
def get_supported_lora_modules(model: nn.Module) -> list[str]:
"""
In vLLM, all linear layers support LoRA.
"""
"""In vLLM, all linear layers support LoRA."""
supported_lora_modules: set[str] = set()
for name, module in model.named_modules():
# get the embedding modules if the module's embedding_modules
@@ -268,6 +264,7 @@ def is_supported_lora_module(
Returns:
True if the module is supported, False otherwise.
"""
module_suffix = module_name.rsplit(".", 1)[-1]
if module_suffix not in supported_lora_modules:
@@ -305,6 +302,7 @@ def is_in_target_modules(
Returns:
True if the module passes the filter, False otherwise.
"""
if target_modules is None:
return True
@@ -329,22 +327,21 @@ def is_in_target_modules(
def get_adapter_absolute_path(lora_path: str) -> str:
"""
Resolves the given lora_path to an absolute local path.
"""Resolves the given lora_path to an absolute local path.
If the lora_path is identified as a Hugging Face model identifier,
it will download the model and return the local snapshot path.
Otherwise, it treats the lora_path as a local file path and
converts it to an absolute path.
Parameters:
lora_path (str): The path to the lora model, which can be an absolute path,
a relative path, or a Hugging Face model identifier.
Args:
lora_path (str): The path to the lora model, which can be an absolute
path, a relative path, or a Hugging Face model identifier.
Returns:
str: The resolved absolute local path to the lora model.
"""
str: The resolved absolute local path to the lora model.
"""
# Check if the path is an absolute path. Return it no matter exists or not.
if os.path.isabs(lora_path):
return lora_path
+6 -3
View File
@@ -28,7 +28,8 @@ class WorkerLoRAManager:
"""WorkerLoRAManager that manages LoRA models on the worker side.
Every request, the requested LoRAs will be loaded (unless they are already
loaded), and every other LoRA will be unloaded."""
loaded), and every other LoRA will be unloaded.
"""
_manager_cls: type[LoRAModelManager] = LoRAModelManager
@@ -74,7 +75,8 @@ class WorkerLoRAManager:
@contextmanager
def dummy_lora_cache(self):
"""Use this context manager to reuse the dummy lora model
to avoid creating it repeatedly."""
to avoid creating it repeatedly.
"""
self._cached_dummy_lora = None
yield
self._cached_dummy_lora = False
@@ -249,7 +251,8 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
Uses an LRU Cache. Every request, the requested LoRAs will be loaded
(unless they are already loaded) and least recently used LoRAs will
be unloaded if the cache is above capacity."""
be unloaded if the cache is above capacity.
"""
_manager_cls: type[LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager