Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+7 -14
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Utilities for Punica kernel construction.
"""
"""Utilities for Punica kernel construction."""
from vllm.triton_utils import tl, triton
@@ -21,8 +19,7 @@ def _accumulate_mm(
group_n: tl.constexpr,
use_fp8_w8a8: tl.constexpr,
):
"""
Core matrix multiplication and accumulation logic with quantization support.
"""Core matrix multiplication and accumulation logic with quantization support.
Args:
tiled_a (tl.tensor): Loaded tile from A matrix
@@ -36,8 +33,8 @@ def _accumulate_mm(
group_k: Block size for K dimension in block-wise quantization
group_n: Block size for N dimension in block-wise quantization
use_fp8_w8a8: Whether using FP8 W8A8 quantization
"""
"""
if use_fp8_w8a8:
if group_k > 0 and group_n > 0:
# Block-wise quantization: scales are loaded per block
@@ -85,8 +82,7 @@ def fp8_mm_k(
USE_GDC: tl.constexpr,
base_k,
):
"""
FP8-compatible matrix multiplication kernel with quantization support.
"""FP8-compatible matrix multiplication kernel with quantization support.
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
B (k x n), iterate through the K dimension to compute the partial/complete
matrix block product with proper dequantization.
@@ -120,6 +116,7 @@ def fp8_mm_k(
b_dtype: datatype of the B matrix
USE_GDC: Whether to use PDL. True indicates use.
base_k (int): Base offset along K dimension for current SPLIT_K group
"""
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
@@ -238,13 +235,11 @@ def do_shrink_kernel_fp8(
per_channel_quant: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""
Given an array of integers that identifies the rows of A, ram,
"""Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice, compute the
matrix product and store in the appropriate output location.
"""
# Identify the lora_ptr from slice_id.
if SLICE_NUM == 1:
cur_lora_ptr = lora_ptr
@@ -432,8 +427,7 @@ def do_expand_kernel_fp8(
use_fp8_w8a8: tl.constexpr,
per_channel_quant: tl.constexpr,
):
"""
FP8-compatible expand kernel for LoRA.
"""FP8-compatible expand kernel for LoRA.
Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice,
@@ -449,7 +443,6 @@ def do_expand_kernel_fp8(
- Per-channel quantization
- Tensor-wise quantization
"""
# ls_d*_ptr can be either an integer or a pointer
if SAME_STRIDE:
cur_lora_d0_stride = ls_d0_ptr
@@ -24,7 +24,7 @@ def _get_lora_id(
top_k_num,
naive_block_assignment: tl.constexpr,
):
"""Returns lora_id"""
"""Returns lora_id."""
if naive_block_assignment:
token_idx = pid_m // top_k_num
return tl.load(token_lora_mapping_ptr + token_idx)
@@ -41,7 +41,7 @@ def _get_expert_id(
max_loras,
naive_block_assignment: tl.constexpr,
):
"""Returns expert_id"""
"""Returns expert_id."""
if naive_block_assignment:
return tl.load(expert_ids_ptr + pid_m)
else:
@@ -61,7 +61,7 @@ def _get_token_offs(
naive_block_assignment: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
):
"""Returns token offsets"""
"""Returns token offsets."""
if naive_block_assignment:
return tl.where(offs == 0, pid_m, num_valid_tokens)
else:
@@ -76,8 +76,7 @@ _LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
"""
`_LORA_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
@@ -101,9 +100,7 @@ def _adjust_kernel_inputs(
sorted_token_ids: torch.Tensor | None,
expert_ids: torch.Tensor,
):
"""
helper function to adjust kernel inputs when sorted_token_ids is None
"""
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
if sorted_token_ids is None:
stride_tl = 0
stride_el = 0
@@ -25,7 +25,7 @@ def _get_lora_id(
top_k_num,
naive_block_assignment: tl.constexpr,
):
"""Returns lora_id"""
"""Returns lora_id."""
if naive_block_assignment:
token_idx = pid_m // top_k_num
return tl.load(token_lora_mapping_ptr + token_idx)
@@ -42,7 +42,7 @@ def _get_expert_id(
max_loras,
naive_block_assignment: tl.constexpr,
):
"""Returns expert_id"""
"""Returns expert_id."""
if naive_block_assignment:
return tl.load(expert_ids_ptr + pid_m)
else:
@@ -62,7 +62,7 @@ def _get_token_offs(
naive_block_assignment: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
):
"""Returns token offsets"""
"""Returns token offsets."""
if naive_block_assignment:
return tl.where(offs == 0, pid_m, num_valid_tokens)
else:
@@ -852,8 +852,7 @@ def _run_fused_moe_lora_small_batch(
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
"""
`_LORA_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
@@ -877,9 +876,7 @@ def _adjust_kernel_inputs(
sorted_token_ids: torch.Tensor | None,
expert_ids: torch.Tensor,
):
"""
helper function to adjust kernel inputs when sorted_token_ids is None
"""
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
if sorted_token_ids is None:
stride_tl = 0
stride_el = 0
+15 -19
View File
@@ -1,19 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Utilities for Punica kernel construction.
"""
"""Utilities for Punica kernel construction."""
from vllm.triton_utils import tl, triton
@triton.jit
def mm_k(
a_ptr,
b_ptr,
ak_stride,
bk_stride,
offset_k,
a_ptr: tl.tensor,
b_ptr: tl.tensor,
ak_stride: int,
bk_stride: int,
offset_k: tl.tensor,
K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
@@ -23,32 +21,34 @@ def mm_k(
CAST_TYPE: tl.constexpr,
b_dtype: tl.constexpr,
USE_GDC: tl.constexpr,
base_k,
base_k: int,
):
"""
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
"""Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
B (k x n), iterate, through the K dimension to compute the partial/complete
matrix block product.
If SPLIT_K == 1, the output m x n product is complete.
If SPLIT_K > 1, the thread block computes partial outputs. The partial
outputs are then atomically summed in the caller code.
Args:
a_ptr: Array of pointers, identifying rows of A
b_ptr: Array of pointers, identifying columns of B
ak_stride: K dimension stride of the A matrix
bk_stride: K dimension stride of the B matrix
offset_k: Offsets within the current BLOCK_K tile along K
K: Length of the K dimension
BLOCK_M: M dimension of the output block m x n
BLOCK_N: N dimension of the output block m x n
BLOCK_K: K dimension atom
EVEN_K: True if the blocks of A and B can be loaded without any
masking.
masking.
SPLIT_K: Parameter signifying parallelism in the K dimension.
CAST_TYPE: if True, cast the values from the A matrix to the B
matrix dtype.
matrix dtype.
b_dtype: datatype of the B matrix
USE_GDC: Whether to use PDL. True indicates use.
base_k: Base offset along K dimension for current SPLIT_K group
"""
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
@@ -142,15 +142,13 @@ def do_expand_kernel(
ADD_INPUTS: tl.constexpr,
USE_GDC: tl.constexpr,
):
"""
Given an array of integers that identifies the rows of A, ram,
"""Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice,
compute the matrix product and store in the appropriate output location.
Given that this is an expand kernel, we don't perform any split-K reduction
as the K dimension is assumed to be small.
"""
# ls_d*_ptr can be either an integer or a pointer
if SAME_STRIDE:
# integer
@@ -268,13 +266,11 @@ def do_shrink_kernel(
SLICE_NUM: tl.constexpr,
USE_GDC: tl.constexpr,
):
"""
Given an array of integers that identifies the rows of A, ram,
"""Given an array of integers that identifies the rows of A, ram,
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
a slice_id that identifies the input/output slice, compute the
matrix product and store in the appropriate output location.
"""
# Identify the lora_ptr from slice_id.
if SLICE_NUM == 1:
# current lora ptr
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -21,8 +20,7 @@ _EXPAND_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
def _get_expand_lora_scale_ptr(lora_weights: list[torch.Tensor], device: torch.device):
"""
`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
"""`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
`profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
@@ -91,9 +89,7 @@ def _lora_expand_kernel_fp8(
per_channel_quant: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""
FP8-compatible expand kernel wrapper.
"""
"""FP8-compatible expand kernel wrapper."""
cta_n_num = tl.cdiv(N, BLOCK_N)
cta_m_num = tl.cdiv(M, BLOCK_M)
@@ -192,8 +188,7 @@ def _lora_expand_fp8(
use_fp8_w8a8: bool = False,
per_channel_quant: bool = False,
) -> None:
"""
FP8-compatible LoRA expand operation.
"""FP8-compatible LoRA expand operation.
Args:
inputs: Input tensor from shrink operation [num_slices, num_tokens, lora_rank]
@@ -206,6 +201,8 @@ def _lora_expand_fp8(
num_tokens_per_lora: Number of tokens per LoRA
lora_token_start_loc: Start location for each LoRA's tokens
lora_ids: LoRA IDs to process
num_active_loras: Number of active LoRAs. Accepted for API parity with
the non-FP8 kernel and unused here.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
offset_start (int, optional): Offset start for output_tensor.
@@ -216,6 +213,7 @@ def _lora_expand_fp8(
group_n (int, optional): Block size for N in block-wise quantization.
use_fp8_w8a8 (bool, optional): Whether to use FP8 W8A8 quantization.
per_channel_quant (bool, optional): Whether to use per-channel quantization.
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
+25 -27
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -147,32 +146,31 @@ def _lora_expand(
offset_start: int = 0,
add_inputs: bool = False,
) -> None:
"""
Args:
inputs (torch.Tensor): input tensor
lora_b_weights (list[torch.Tensor]): lora'b weight
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
offset_start (int, optional): Offset start for output_tensor.
Defaults to 0.
add_inputs (bool, optional): Whether to add the input tensor to the
output tensor. Defaults to False.
"""
"""Args:
inputs (torch.Tensor): input tensor
lora_b_weights (list[torch.Tensor]): lora'b weight
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
offset_start (int, optional): Offset start for output_tensor.
Defaults to 0.
add_inputs (bool, optional): Whether to add the input tensor to the
output tensor. Defaults to False.
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
# None of the inputs require LoRA.
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
LoRA kernels metadata preparation utilities.
"""
"""LoRA kernels metadata preparation utilities."""
import bisect
from dataclasses import dataclass, field
@@ -107,14 +105,13 @@ class LoRAKernelMeta:
self.num_active_loras_cpu.fill_(0)
def prepare_tensors(self, token_lora_mapping: torch.Tensor) -> None:
"""
Prepare kernel metadata tensors for the current forward pass.
"""Prepare kernel metadata tensors for the current forward pass.
Args:
token_lora_mapping (torch.Tensor): Tensor containing lora indices
for each input token.
"""
"""
self._reset()
# Check and record no-lora case.
@@ -180,8 +177,7 @@ class LoRAKernelMeta:
torch.Tensor,
torch.Tensor,
]:
"""
This function returns the kernel metadata required for the current
"""This function returns the kernel metadata required for the current
forward pass execution of the kernel. The function returns all the
metadata required by the kernel, in order, as a tuple, so it can be
unpacked directly during the lora_shrink/lora_expand function call.
@@ -189,6 +185,10 @@ class LoRAKernelMeta:
Args:
token_nums (int): Number of input tokens in the current forward
pass of the kernel.
specialize_active_lora (bool): If True, report the actual active
LoRA count; otherwise report the padded default, which keeps
the kernel launch shape stable.
"""
if specialize_active_lora:
num_active_loras = self.num_active_loras_cpu
+19 -21
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -20,8 +19,7 @@ _SHRINK_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], tuple] = {}
def _get_shrink_lora_scale_ptr(
lora_scale_weights: list[torch.Tensor], device: torch.device
):
"""
`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
"""`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
`profile_run`. After this, it remains constant and subsequent usage is
through LUT.
@@ -241,23 +239,23 @@ def _lora_shrink_fp8(
use_fp8_w8a8: bool = False,
per_channel_quant: bool = False,
) -> None:
"""
Args:
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
output_tensor: Output tensor (FP16/BF16/FP32)
token_lora_mapping: Token to LoRA ID mapping
token_indices_sorted_by_lora_ids: Sorted token indices
num_tokens_per_lora: Number of tokens per LoRA
lora_token_start_loc: Start location for each LoRA's tokens
lora_ids: LoRA IDs to process
scaling: LoRA scaling factor
a_scale: Activation quantization scales
b_scale: Weight quantization scales per slice
group_k: Block size for K dimension quantization
group_n: Block size for N dimension quantization
use_fp8_w8a8: Whether to use FP8 weights and activations
per_channel_quant: Whether to use per-channel quantization
"""Args:
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
output_tensor: Output tensor (FP16/BF16/FP32)
token_lora_mapping: Token to LoRA ID mapping
token_indices_sorted_by_lora_ids: Sorted token indices
num_tokens_per_lora: Number of tokens per LoRA
lora_token_start_loc: Start location for each LoRA's tokens
lora_ids: LoRA IDs to process
scaling: LoRA scaling factor
a_scale: Activation quantization scales
b_scale: Weight quantization scales per slice
group_k: Block size for K dimension quantization
group_n: Block size for N dimension quantization
use_fp8_w8a8: Whether to use FP8 weights and activations
per_channel_quant: Whether to use per-channel quantization
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
+25 -27
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Based on:
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
@@ -142,32 +141,31 @@ def _lora_shrink(
num_active_loras: torch.Tensor, # CPU tensor [1], number of active LoRAs
scaling: float,
) -> None:
"""
Args:
inputs (torch.Tensor): Input tensor
lora_a_weights (list[torch.Tensor]): LoRA weights
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
number of active LoRAs. Stored as a tensor (not int) so
torch.compile treats it as dynamic rather than a constant.
scaling (float): Scaling factor.
"""
"""Args:
inputs (torch.Tensor): Input tensor
lora_a_weights (list[torch.Tensor]): LoRA weights
output_tensor (torch.Tensor): output tensor
token_lora_mapping (torch.Tensor): A tensor mapping each input token
to the lora-id related to that token. A value of -1 indicates that
LoRA doesn't apply to that token.
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
the A matrix grouped by LoRA IDs.
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
of tokens that are to be processed by LoRA ID lora_ids[i]
lora_token_start_loc (torch.Tensor): A cumulative sum of
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
lora_token_start_loc[i], along with num_tokens_per_lora[i]
identifies the region in token_indices_sorted_by_lora_ids that
LoRA lora_ids[i] should process.
lora_ids (torch.Tensor): LoRA ids to process.
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
if there are any requests that require LoRA.
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
number of active LoRAs. Stored as a tensor (not int) so
torch.compile treats it as dynamic rather than a constant.
scaling (float): Scaling factor.
"""
assert no_lora_flag_cpu.numel() == 1
if no_lora_flag_cpu.item():
# None of the inputs require LoRA.
+3 -8
View File
@@ -23,8 +23,7 @@ _LORA_B_PTR_DICT: dict[tuple[int, ...], tuple[torch.tensor, ...]] = {}
def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
"""
`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
@@ -75,14 +74,12 @@ def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
def _get_lora_b_ptr(
lora_weights: list[torch.Tensor], offset_start: int, device: torch.device
):
"""
`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
"""`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
After this, it remains constant and subsequent usage is through LUT.
Refer to:
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
"""
key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights)
if values := _LORA_B_PTR_DICT.get(key):
return values
@@ -317,9 +314,7 @@ def get_lora_op_configs(
@lru_cache
def supports_pdl(device: torch.device | None = None) -> bool:
"""
Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py
"""
"""Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py."""
# PDL requires compute capability SM90 or above
return (