Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utilities for Punica kernel construction.
|
||||
"""
|
||||
"""Utilities for Punica kernel construction."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@@ -21,8 +19,7 @@ def _accumulate_mm(
|
||||
group_n: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Core matrix multiplication and accumulation logic with quantization support.
|
||||
"""Core matrix multiplication and accumulation logic with quantization support.
|
||||
|
||||
Args:
|
||||
tiled_a (tl.tensor): Loaded tile from A matrix
|
||||
@@ -36,8 +33,8 @@ def _accumulate_mm(
|
||||
group_k: Block size for K dimension in block-wise quantization
|
||||
group_n: Block size for N dimension in block-wise quantization
|
||||
use_fp8_w8a8: Whether using FP8 W8A8 quantization
|
||||
"""
|
||||
|
||||
"""
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise quantization: scales are loaded per block
|
||||
@@ -85,8 +82,7 @@ def fp8_mm_k(
|
||||
USE_GDC: tl.constexpr,
|
||||
base_k,
|
||||
):
|
||||
"""
|
||||
FP8-compatible matrix multiplication kernel with quantization support.
|
||||
"""FP8-compatible matrix multiplication kernel with quantization support.
|
||||
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
B (k x n), iterate through the K dimension to compute the partial/complete
|
||||
matrix block product with proper dequantization.
|
||||
@@ -120,6 +116,7 @@ def fp8_mm_k(
|
||||
b_dtype: datatype of the B matrix
|
||||
USE_GDC: Whether to use PDL. True indicates use.
|
||||
base_k (int): Base offset along K dimension for current SPLIT_K group
|
||||
|
||||
"""
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
@@ -238,13 +235,11 @@ def do_shrink_kernel_fp8(
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
"""Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice, compute the
|
||||
matrix product and store in the appropriate output location.
|
||||
"""
|
||||
|
||||
# Identify the lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
cur_lora_ptr = lora_ptr
|
||||
@@ -432,8 +427,7 @@ def do_expand_kernel_fp8(
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
FP8-compatible expand kernel for LoRA.
|
||||
"""FP8-compatible expand kernel for LoRA.
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice,
|
||||
@@ -449,7 +443,6 @@ def do_expand_kernel_fp8(
|
||||
- Per-channel quantization
|
||||
- Tensor-wise quantization
|
||||
"""
|
||||
|
||||
# ls_d*_ptr can be either an integer or a pointer
|
||||
if SAME_STRIDE:
|
||||
cur_lora_d0_stride = ls_d0_ptr
|
||||
|
||||
@@ -24,7 +24,7 @@ def _get_lora_id(
|
||||
top_k_num,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns lora_id"""
|
||||
"""Returns lora_id."""
|
||||
if naive_block_assignment:
|
||||
token_idx = pid_m // top_k_num
|
||||
return tl.load(token_lora_mapping_ptr + token_idx)
|
||||
@@ -41,7 +41,7 @@ def _get_expert_id(
|
||||
max_loras,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns expert_id"""
|
||||
"""Returns expert_id."""
|
||||
if naive_block_assignment:
|
||||
return tl.load(expert_ids_ptr + pid_m)
|
||||
else:
|
||||
@@ -61,7 +61,7 @@ def _get_token_offs(
|
||||
naive_block_assignment: tl.constexpr,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
):
|
||||
"""Returns token offsets"""
|
||||
"""Returns token offsets."""
|
||||
if naive_block_assignment:
|
||||
return tl.where(offs == 0, pid_m, num_valid_tokens)
|
||||
else:
|
||||
@@ -76,8 +76,7 @@ _LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
|
||||
|
||||
|
||||
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
@@ -101,9 +100,7 @@ def _adjust_kernel_inputs(
|
||||
sorted_token_ids: torch.Tensor | None,
|
||||
expert_ids: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
helper function to adjust kernel inputs when sorted_token_ids is None
|
||||
"""
|
||||
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
|
||||
if sorted_token_ids is None:
|
||||
stride_tl = 0
|
||||
stride_el = 0
|
||||
|
||||
@@ -25,7 +25,7 @@ def _get_lora_id(
|
||||
top_k_num,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns lora_id"""
|
||||
"""Returns lora_id."""
|
||||
if naive_block_assignment:
|
||||
token_idx = pid_m // top_k_num
|
||||
return tl.load(token_lora_mapping_ptr + token_idx)
|
||||
@@ -42,7 +42,7 @@ def _get_expert_id(
|
||||
max_loras,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns expert_id"""
|
||||
"""Returns expert_id."""
|
||||
if naive_block_assignment:
|
||||
return tl.load(expert_ids_ptr + pid_m)
|
||||
else:
|
||||
@@ -62,7 +62,7 @@ def _get_token_offs(
|
||||
naive_block_assignment: tl.constexpr,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
):
|
||||
"""Returns token offsets"""
|
||||
"""Returns token offsets."""
|
||||
if naive_block_assignment:
|
||||
return tl.where(offs == 0, pid_m, num_valid_tokens)
|
||||
else:
|
||||
@@ -852,8 +852,7 @@ def _run_fused_moe_lora_small_batch(
|
||||
|
||||
|
||||
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
@@ -877,9 +876,7 @@ def _adjust_kernel_inputs(
|
||||
sorted_token_ids: torch.Tensor | None,
|
||||
expert_ids: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
helper function to adjust kernel inputs when sorted_token_ids is None
|
||||
"""
|
||||
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
|
||||
if sorted_token_ids is None:
|
||||
stride_tl = 0
|
||||
stride_el = 0
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utilities for Punica kernel construction.
|
||||
"""
|
||||
"""Utilities for Punica kernel construction."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
ak_stride,
|
||||
bk_stride,
|
||||
offset_k,
|
||||
a_ptr: tl.tensor,
|
||||
b_ptr: tl.tensor,
|
||||
ak_stride: int,
|
||||
bk_stride: int,
|
||||
offset_k: tl.tensor,
|
||||
K: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
@@ -23,32 +21,34 @@ def mm_k(
|
||||
CAST_TYPE: tl.constexpr,
|
||||
b_dtype: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
base_k,
|
||||
base_k: int,
|
||||
):
|
||||
"""
|
||||
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
"""Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
B (k x n), iterate, through the K dimension to compute the partial/complete
|
||||
matrix block product.
|
||||
If SPLIT_K == 1, the output m x n product is complete.
|
||||
If SPLIT_K > 1, the thread block computes partial outputs. The partial
|
||||
outputs are then atomically summed in the caller code.
|
||||
|
||||
Args:
|
||||
a_ptr: Array of pointers, identifying rows of A
|
||||
b_ptr: Array of pointers, identifying columns of B
|
||||
ak_stride: K dimension stride of the A matrix
|
||||
bk_stride: K dimension stride of the B matrix
|
||||
offset_k: Offsets within the current BLOCK_K tile along K
|
||||
K: Length of the K dimension
|
||||
BLOCK_M: M dimension of the output block m x n
|
||||
BLOCK_N: N dimension of the output block m x n
|
||||
BLOCK_K: K dimension atom
|
||||
EVEN_K: True if the blocks of A and B can be loaded without any
|
||||
masking.
|
||||
masking.
|
||||
SPLIT_K: Parameter signifying parallelism in the K dimension.
|
||||
CAST_TYPE: if True, cast the values from the A matrix to the B
|
||||
matrix dtype.
|
||||
matrix dtype.
|
||||
b_dtype: datatype of the B matrix
|
||||
USE_GDC: Whether to use PDL. True indicates use.
|
||||
base_k: Base offset along K dimension for current SPLIT_K group
|
||||
|
||||
"""
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
@@ -142,15 +142,13 @@ def do_expand_kernel(
|
||||
ADD_INPUTS: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
"""Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice,
|
||||
compute the matrix product and store in the appropriate output location.
|
||||
Given that this is an expand kernel, we don't perform any split-K reduction
|
||||
as the K dimension is assumed to be small.
|
||||
"""
|
||||
|
||||
# ls_d*_ptr can be either an integer or a pointer
|
||||
if SAME_STRIDE:
|
||||
# integer
|
||||
@@ -268,13 +266,11 @@ def do_shrink_kernel(
|
||||
SLICE_NUM: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
"""Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice, compute the
|
||||
matrix product and store in the appropriate output location.
|
||||
"""
|
||||
|
||||
# Identify the lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
# current lora ptr
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -21,8 +20,7 @@ _EXPAND_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
|
||||
|
||||
|
||||
def _get_expand_lora_scale_ptr(lora_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
"""`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
`profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
@@ -91,9 +89,7 @@ def _lora_expand_kernel_fp8(
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
FP8-compatible expand kernel wrapper.
|
||||
"""
|
||||
"""FP8-compatible expand kernel wrapper."""
|
||||
cta_n_num = tl.cdiv(N, BLOCK_N)
|
||||
cta_m_num = tl.cdiv(M, BLOCK_M)
|
||||
|
||||
@@ -192,8 +188,7 @@ def _lora_expand_fp8(
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
FP8-compatible LoRA expand operation.
|
||||
"""FP8-compatible LoRA expand operation.
|
||||
|
||||
Args:
|
||||
inputs: Input tensor from shrink operation [num_slices, num_tokens, lora_rank]
|
||||
@@ -206,6 +201,8 @@ def _lora_expand_fp8(
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
num_active_loras: Number of active LoRAs. Accepted for API parity with
|
||||
the non-FP8 kernel and unused here.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
offset_start (int, optional): Offset start for output_tensor.
|
||||
@@ -216,6 +213,7 @@ def _lora_expand_fp8(
|
||||
group_n (int, optional): Block size for N in block-wise quantization.
|
||||
use_fp8_w8a8 (bool, optional): Whether to use FP8 W8A8 quantization.
|
||||
per_channel_quant (bool, optional): Whether to use per-channel quantization.
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -147,32 +146,31 @@ def _lora_expand(
|
||||
offset_start: int = 0,
|
||||
add_inputs: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
inputs (torch.Tensor): input tensor
|
||||
lora_b_weights (list[torch.Tensor]): lora'b weight
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
offset_start (int, optional): Offset start for output_tensor.
|
||||
Defaults to 0.
|
||||
add_inputs (bool, optional): Whether to add the input tensor to the
|
||||
output tensor. Defaults to False.
|
||||
"""
|
||||
"""Args:
|
||||
inputs (torch.Tensor): input tensor
|
||||
lora_b_weights (list[torch.Tensor]): lora'b weight
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
offset_start (int, optional): Offset start for output_tensor.
|
||||
Defaults to 0.
|
||||
add_inputs (bool, optional): Whether to add the input tensor to the
|
||||
output tensor. Defaults to False.
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
# None of the inputs require LoRA.
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
LoRA kernels metadata preparation utilities.
|
||||
"""
|
||||
"""LoRA kernels metadata preparation utilities."""
|
||||
|
||||
import bisect
|
||||
from dataclasses import dataclass, field
|
||||
@@ -107,14 +105,13 @@ class LoRAKernelMeta:
|
||||
self.num_active_loras_cpu.fill_(0)
|
||||
|
||||
def prepare_tensors(self, token_lora_mapping: torch.Tensor) -> None:
|
||||
"""
|
||||
Prepare kernel metadata tensors for the current forward pass.
|
||||
"""Prepare kernel metadata tensors for the current forward pass.
|
||||
|
||||
Args:
|
||||
token_lora_mapping (torch.Tensor): Tensor containing lora indices
|
||||
for each input token.
|
||||
"""
|
||||
|
||||
"""
|
||||
self._reset()
|
||||
|
||||
# Check and record no-lora case.
|
||||
@@ -180,8 +177,7 @@ class LoRAKernelMeta:
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
"""
|
||||
This function returns the kernel metadata required for the current
|
||||
"""This function returns the kernel metadata required for the current
|
||||
forward pass execution of the kernel. The function returns all the
|
||||
metadata required by the kernel, in order, as a tuple, so it can be
|
||||
unpacked directly during the lora_shrink/lora_expand function call.
|
||||
@@ -189,6 +185,10 @@ class LoRAKernelMeta:
|
||||
Args:
|
||||
token_nums (int): Number of input tokens in the current forward
|
||||
pass of the kernel.
|
||||
specialize_active_lora (bool): If True, report the actual active
|
||||
LoRA count; otherwise report the padded default, which keeps
|
||||
the kernel launch shape stable.
|
||||
|
||||
"""
|
||||
if specialize_active_lora:
|
||||
num_active_loras = self.num_active_loras_cpu
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -20,8 +19,7 @@ _SHRINK_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], tuple] = {}
|
||||
def _get_shrink_lora_scale_ptr(
|
||||
lora_scale_weights: list[torch.Tensor], device: torch.device
|
||||
):
|
||||
"""
|
||||
`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
"""`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
`profile_run`. After this, it remains constant and subsequent usage is
|
||||
through LUT.
|
||||
|
||||
@@ -241,23 +239,23 @@ def _lora_shrink_fp8(
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
|
||||
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
|
||||
output_tensor: Output tensor (FP16/BF16/FP32)
|
||||
token_lora_mapping: Token to LoRA ID mapping
|
||||
token_indices_sorted_by_lora_ids: Sorted token indices
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
scaling: LoRA scaling factor
|
||||
a_scale: Activation quantization scales
|
||||
b_scale: Weight quantization scales per slice
|
||||
group_k: Block size for K dimension quantization
|
||||
group_n: Block size for N dimension quantization
|
||||
use_fp8_w8a8: Whether to use FP8 weights and activations
|
||||
per_channel_quant: Whether to use per-channel quantization
|
||||
"""Args:
|
||||
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
|
||||
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
|
||||
output_tensor: Output tensor (FP16/BF16/FP32)
|
||||
token_lora_mapping: Token to LoRA ID mapping
|
||||
token_indices_sorted_by_lora_ids: Sorted token indices
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
scaling: LoRA scaling factor
|
||||
a_scale: Activation quantization scales
|
||||
b_scale: Weight quantization scales per slice
|
||||
group_k: Block size for K dimension quantization
|
||||
group_n: Block size for N dimension quantization
|
||||
use_fp8_w8a8: Whether to use FP8 weights and activations
|
||||
per_channel_quant: Whether to use per-channel quantization
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -142,32 +141,31 @@ def _lora_shrink(
|
||||
num_active_loras: torch.Tensor, # CPU tensor [1], number of active LoRAs
|
||||
scaling: float,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
inputs (torch.Tensor): Input tensor
|
||||
lora_a_weights (list[torch.Tensor]): LoRA weights
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
|
||||
number of active LoRAs. Stored as a tensor (not int) so
|
||||
torch.compile treats it as dynamic rather than a constant.
|
||||
scaling (float): Scaling factor.
|
||||
"""
|
||||
"""Args:
|
||||
inputs (torch.Tensor): Input tensor
|
||||
lora_a_weights (list[torch.Tensor]): LoRA weights
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
|
||||
number of active LoRAs. Stored as a tensor (not int) so
|
||||
torch.compile treats it as dynamic rather than a constant.
|
||||
scaling (float): Scaling factor.
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
# None of the inputs require LoRA.
|
||||
|
||||
@@ -23,8 +23,7 @@ _LORA_B_PTR_DICT: dict[tuple[int, ...], tuple[torch.tensor, ...]] = {}
|
||||
|
||||
|
||||
def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
@@ -75,14 +74,12 @@ def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
|
||||
def _get_lora_b_ptr(
|
||||
lora_weights: list[torch.Tensor], offset_start: int, device: torch.device
|
||||
):
|
||||
"""
|
||||
`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
|
||||
"""
|
||||
|
||||
key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights)
|
||||
if values := _LORA_B_PTR_DICT.get(key):
|
||||
return values
|
||||
@@ -317,9 +314,7 @@ def get_lora_op_configs(
|
||||
|
||||
@lru_cache
|
||||
def supports_pdl(device: torch.device | None = None) -> bool:
|
||||
"""
|
||||
Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py
|
||||
"""
|
||||
"""Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py."""
|
||||
# PDL requires compute capability SM90 or above
|
||||
|
||||
return (
|
||||
|
||||
Reference in New Issue
Block a user