Fp8 lora dense kernel (#35242)
Signed-off-by: Yu Gong <yu3.gong@gmail.com>
This commit is contained in:
@@ -12,13 +12,17 @@ from vllm.lora.ops.triton_ops.fused_moe_lora_op import (
|
||||
fused_moe_lora_expand,
|
||||
fused_moe_lora_shrink,
|
||||
)
|
||||
from vllm.lora.ops.triton_ops.lora_expand_fp8_op import lora_expand_fp8
|
||||
from vllm.lora.ops.triton_ops.lora_expand_op import lora_expand
|
||||
from vllm.lora.ops.triton_ops.lora_kernel_metadata import LoRAKernelMeta
|
||||
from vllm.lora.ops.triton_ops.lora_shrink_fp8_op import lora_shrink_fp8
|
||||
from vllm.lora.ops.triton_ops.lora_shrink_op import lora_shrink
|
||||
|
||||
__all__ = [
|
||||
"lora_expand",
|
||||
"lora_expand_fp8",
|
||||
"lora_shrink",
|
||||
"lora_shrink_fp8",
|
||||
"LoRAKernelMeta",
|
||||
"fused_moe_lora",
|
||||
"fused_moe_lora_shrink",
|
||||
|
||||
@@ -0,0 +1,603 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utilities for Punica kernel construction.
|
||||
"""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _accumulate_mm(
|
||||
tiled_a,
|
||||
tiled_b,
|
||||
accumulator,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
iter_k,
|
||||
group_k: tl.constexpr,
|
||||
group_n: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Core matrix multiplication and accumulation logic with quantization support.
|
||||
|
||||
Args:
|
||||
tiled_a (tl.tensor): Loaded tile from A matrix
|
||||
tiled_b (tl.tensor): Loaded tile from B matrix
|
||||
accumulator (tl.tensor): Current accumulator value
|
||||
a_scale_ptr (tl.tensor): Scale pointer for A matrix
|
||||
b_scale_ptr (tl.tensor): Scale pointer for B matrix
|
||||
a_scale_k_stride (int): K dimension stride for A's block-wise scales
|
||||
b_scale_k_stride (int): K dimension stride for B's block-wise scales
|
||||
iter_k (int): Current iteration's global K offset
|
||||
group_k: Block size for K dimension in block-wise quantization
|
||||
group_n: Block size for N dimension in block-wise quantization
|
||||
use_fp8_w8a8: Whether using FP8 W8A8 quantization
|
||||
"""
|
||||
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise quantization: scales are loaded per block
|
||||
offs_ks = iter_k // group_k
|
||||
# a_scale_ptr is (BLOCK_M,) tensor of base pointers per row
|
||||
# Load scale for current K-group, result shape: (BLOCK_M,)
|
||||
a_scale = tl.load(a_scale_ptr + offs_ks * a_scale_k_stride)
|
||||
# b_scale_ptr is (BLOCK_N,) tensor with N-offset pre-baked
|
||||
# Load scale for current K-group, result shape: (BLOCK_N,)
|
||||
b_scale = tl.load(b_scale_ptr + offs_ks * b_scale_k_stride)
|
||||
accumulator += (
|
||||
tl.dot(tiled_a, tiled_b) * a_scale[:, None] * b_scale[None, :]
|
||||
)
|
||||
else:
|
||||
# Tensor-wise or per-channel: accumulate and scale at end
|
||||
accumulator = tl.dot(tiled_a, tiled_b, acc=accumulator)
|
||||
else:
|
||||
accumulator += tl.dot(tiled_a, tiled_b)
|
||||
return accumulator
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fp8_mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
ak_stride,
|
||||
bk_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
offset_k,
|
||||
K: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
group_n: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
CAST_TYPE: tl.constexpr,
|
||||
b_dtype: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
base_k,
|
||||
):
|
||||
"""
|
||||
FP8-compatible matrix multiplication kernel with quantization support.
|
||||
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
B (k x n), iterate through the K dimension to compute the partial/complete
|
||||
matrix block product with proper dequantization.
|
||||
|
||||
Args:
|
||||
a_ptr (tl.tensor): Array of pointers, identifying rows of A
|
||||
(FP8 or other dtype)
|
||||
b_ptr (tl.tensor): Array of pointers, identifying columns of B
|
||||
(FP8 dtype)
|
||||
a_scale_ptr (tl.tensor): Scale pointer for A matrix
|
||||
(per-token or block-wise)
|
||||
b_scale_ptr (tl.tensor): Scale pointer for B matrix
|
||||
(per-channel or block-wise)
|
||||
ak_stride (int): K dimension stride of the A matrix
|
||||
bk_stride (int): K dimension stride of the B matrix
|
||||
a_scale_k_stride (int): K dimension stride for A's block-wise scales
|
||||
b_scale_k_stride (int): K dimension stride for B's block-wise scales
|
||||
offset_k (int): Base offset along K dimension
|
||||
K: Length of the K dimension
|
||||
BLOCK_M: M dimension of the output block m x n
|
||||
BLOCK_N: N dimension of the output block m x n
|
||||
BLOCK_K: K dimension atom
|
||||
EVEN_K: True if the blocks of A and B can be loaded without masking
|
||||
SPLIT_K: Parameter signifying parallelism in the K dimension
|
||||
group_k: Block size for K dimension in block-wise quantization
|
||||
group_n: Block size for N dimension in block-wise quantization
|
||||
use_fp8_w8a8: Whether using FP8 W8A8 quantization
|
||||
per_channel_quant: Whether using per-channel quantization
|
||||
CAST_TYPE: if True, cast the values from the A matrix to the B
|
||||
matrix dtype.
|
||||
b_dtype: datatype of the B matrix
|
||||
USE_GDC: Whether to use PDL. True indicates use.
|
||||
base_k (int): Base offset along K dimension for current SPLIT_K group
|
||||
"""
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
# Step size along K for each iteration
|
||||
STEP_K = BLOCK_K * SPLIT_K
|
||||
|
||||
# Total number of iterations (compile-time constant)
|
||||
num_iters = tl.cdiv(K, STEP_K)
|
||||
|
||||
for k in range(num_iters):
|
||||
# Current iteration's global K offset
|
||||
iter_k = k * STEP_K + base_k
|
||||
block_end = iter_k + BLOCK_K
|
||||
|
||||
# Skip iterations that are entirely past the K boundary
|
||||
if not EVEN_K and iter_k >= K:
|
||||
pass
|
||||
elif EVEN_K or block_end <= K:
|
||||
# No masking needed: either K is evenly divisible (EVEN_K)
|
||||
# or this block fits entirely within K
|
||||
tiled_b = tl.load(b_ptr)
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tiled_a = tl.load(a_ptr)
|
||||
if CAST_TYPE:
|
||||
tiled_a = tiled_a.to(b_dtype)
|
||||
|
||||
accumulator = _accumulate_mm(
|
||||
tiled_a,
|
||||
tiled_b,
|
||||
accumulator,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
iter_k,
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
)
|
||||
else:
|
||||
# Partial block at the tail: mask out-of-bounds elements
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
mask = iter_k + k_offsets < K
|
||||
tiled_b = tl.load(b_ptr, mask=mask[:, None], other=0.0)
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tiled_a = tl.load(a_ptr, mask=mask[None, :], other=0.0)
|
||||
if CAST_TYPE:
|
||||
tiled_a = tiled_a.to(b_dtype)
|
||||
|
||||
accumulator = _accumulate_mm(
|
||||
tiled_a,
|
||||
tiled_b,
|
||||
accumulator,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
iter_k,
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
)
|
||||
|
||||
a_ptr += STEP_K * ak_stride
|
||||
b_ptr += STEP_K * bk_stride
|
||||
|
||||
return accumulator
|
||||
|
||||
|
||||
@triton.jit
|
||||
def do_shrink_kernel_fp8(
|
||||
pid_n,
|
||||
pid_sk,
|
||||
slice_id,
|
||||
lora_index,
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
N,
|
||||
K,
|
||||
M_LEN,
|
||||
ram,
|
||||
# input strides
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
# lora strides
|
||||
lora_d0_stride,
|
||||
lora_d1_stride,
|
||||
lora_d2_stride,
|
||||
# scale strides
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
# output strides
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
output_d2_stride,
|
||||
scaling,
|
||||
# block size for block-wise quantization
|
||||
group_n: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
SLICE_NUM: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice, compute the
|
||||
matrix product and store in the appropriate output location.
|
||||
"""
|
||||
|
||||
# Identify the lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
cur_lora_ptr = lora_ptr
|
||||
cur_b_scale_ptr = b_scale_ptr
|
||||
else:
|
||||
cur_lora_ptr = (
|
||||
tl.load(lora_ptr + slice_id).to(tl.pointer_type(tl.float8e4nv))
|
||||
if b_scale_ptr is not None
|
||||
else tl.load(lora_ptr + slice_id).to(
|
||||
tl.pointer_type(input_ptr.dtype.element_ty)
|
||||
)
|
||||
)
|
||||
cur_b_scale_ptr = (
|
||||
tl.load(b_scale_ptr + slice_id).to(tl.pointer_type(tl.float32))
|
||||
if b_scale_ptr is not None
|
||||
else b_scale_ptr
|
||||
)
|
||||
|
||||
# Identify the column indices of B to process.
|
||||
offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
|
||||
rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
|
||||
|
||||
# Identify A and B block pointers
|
||||
offset_k = pid_sk * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
a_ptr = (
|
||||
input_ptr + ram[:, None] * input_d0_stride + offset_k[None, :] * input_d1_stride
|
||||
)
|
||||
b_ptr = (
|
||||
cur_lora_ptr
|
||||
+ lora_d0_stride * lora_index
|
||||
+ rbn[None, :] * lora_d1_stride
|
||||
+ offset_k[:, None] * lora_d2_stride
|
||||
)
|
||||
|
||||
# Load scales for tensor-wise or per-channel quantization (outside the loop)
|
||||
# Block-wise scales are loaded inside fp8_mm_k
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise: compute scale pointers for fp8_mm_k
|
||||
# a_scale: per-row base pointers, shape (BLOCK_M,)
|
||||
# Each pointer points to the start of that row's scale data
|
||||
mm_a_scale_ptr = a_scale_ptr + ram * a_scale_m_stride
|
||||
|
||||
# b_scale: pre-compute N-dimension offset
|
||||
# We need to bake in the N-group offset since fp8_mm_k doesn't know pid_n
|
||||
n_offset = pid_n * BLOCK_N
|
||||
offs_ns = (n_offset + tl.arange(0, BLOCK_N)) // group_n
|
||||
# Base pointer with lora offset + N-group offset baked in, shape (BLOCK_N,)
|
||||
mm_b_scale_ptr = (
|
||||
cur_b_scale_ptr
|
||||
+ lora_index * b_scale_l_stride
|
||||
+ offs_ns * b_scale_n_stride
|
||||
)
|
||||
elif per_channel_quant:
|
||||
# Per-channel for weights, per-token for activations
|
||||
b_scale_ptrs = (
|
||||
cur_b_scale_ptr + lora_index * b_scale_l_stride + rbn * b_scale_n_stride
|
||||
)
|
||||
b_scale = tl.load(b_scale_ptrs)
|
||||
# Per-token activation scale
|
||||
a_scale = tl.load(a_scale_ptr + ram * a_scale_m_stride)[:, None]
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Tensor-wise quantization
|
||||
a_scale = tl.load(a_scale_ptr) if a_scale_ptr is not None else 1.0
|
||||
b_scale = tl.load(cur_b_scale_ptr + lora_index * b_scale_l_stride)
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Non-quantized path
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
|
||||
# Compute partial/complete block matrix product.
|
||||
accumulator = fp8_mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
mm_a_scale_ptr,
|
||||
mm_b_scale_ptr,
|
||||
input_d1_stride,
|
||||
lora_d2_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
offset_k,
|
||||
K,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
EVEN_K,
|
||||
SPLIT_K,
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
per_channel_quant,
|
||||
False,
|
||||
cur_lora_ptr.dtype.element_ty,
|
||||
USE_GDC,
|
||||
base_k=pid_sk * BLOCK_K,
|
||||
)
|
||||
# GDC launch dependents hints the runtime system to launch dependent kernels.
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# Apply dequantization scales for tensor-wise/per-channel quantization
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise: already applied in fp8_mm_k
|
||||
pass
|
||||
else:
|
||||
# Tensor-wise or per-channel: apply scales after accumulation
|
||||
accumulator = accumulator * a_scale * b_scale
|
||||
|
||||
# Apply LoRA scaling factor
|
||||
accumulator *= scaling
|
||||
|
||||
# Identify the C output pointers to store the results of the accumulator.
|
||||
offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
|
||||
offset_cm = tl.arange(0, BLOCK_M)
|
||||
cur_out_ptr = out_ptr if SLICE_NUM == 1 else out_ptr + slice_id * output_d0_stride
|
||||
c_ptr = (
|
||||
cur_out_ptr
|
||||
+ ram[:, None] * output_d1_stride
|
||||
+ offset_cn[None, :] * output_d2_stride
|
||||
)
|
||||
c_mask = (offset_cm[:, None] < M_LEN) & (offset_cn[None, :] < N)
|
||||
|
||||
# Cast accumulator to output dtype
|
||||
accumulator = accumulator.to(out_ptr.dtype.element_ty)
|
||||
|
||||
# handles write-back with reduction-splitting
|
||||
if SPLIT_K == 1:
|
||||
tl.store(c_ptr, accumulator, mask=c_mask)
|
||||
else:
|
||||
tl.atomic_add(c_ptr, accumulator, mask=c_mask, sem="relaxed")
|
||||
|
||||
|
||||
@triton.jit
|
||||
def do_expand_kernel_fp8(
|
||||
pid_n,
|
||||
lora_index,
|
||||
slice_id,
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
N,
|
||||
K,
|
||||
M_LEN,
|
||||
ram, # array identifying the rows of Input ptr to operate on
|
||||
slice_start_loc,
|
||||
# input ptr strides
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
input_d2_stride,
|
||||
# lora ptr strides
|
||||
ls_d0_ptr,
|
||||
ls_d1_ptr,
|
||||
ls_d2_ptr,
|
||||
# scale strides
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
# out ptr strides
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
# block size for block-wise quantization
|
||||
group_n: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
# constants
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
SAME_STRIDE: tl.constexpr,
|
||||
SLICE_NUM: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
CAST_TYPE: tl.constexpr,
|
||||
ADD_INPUTS: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
FP8-compatible expand kernel for LoRA.
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice,
|
||||
compute the matrix product with FP8 quantization support and store in
|
||||
the appropriate output location.
|
||||
|
||||
For expand kernel, the input (shrink output) may be in FP32/FP16/BF16,
|
||||
while the LoRA B weights can be in FP8.
|
||||
|
||||
Supports:
|
||||
- FP8 W8A8 quantization for LoRA B weights
|
||||
- Block-wise quantization with configurable group_k and group_n
|
||||
- Per-channel quantization
|
||||
- Tensor-wise quantization
|
||||
"""
|
||||
|
||||
# ls_d*_ptr can be either an integer or a pointer
|
||||
if SAME_STRIDE:
|
||||
cur_lora_d0_stride = ls_d0_ptr
|
||||
cur_lora_d1_stride = ls_d1_ptr
|
||||
cur_lora_d2_stride = ls_d2_ptr
|
||||
else:
|
||||
cur_lora_d0_stride = tl.load(ls_d0_ptr + slice_id)
|
||||
cur_lora_d1_stride = tl.load(ls_d1_ptr + slice_id)
|
||||
cur_lora_d2_stride = tl.load(ls_d2_ptr + slice_id)
|
||||
|
||||
# Identify the input_ptr and lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
cur_input_ptr = input_ptr
|
||||
if use_fp8_w8a8:
|
||||
cur_lora_ptr = lora_ptr
|
||||
cur_b_scale_ptr = b_scale_ptr
|
||||
else:
|
||||
cur_lora_ptr = lora_ptr
|
||||
cur_b_scale_ptr = b_scale_ptr # May be None for non-quantized
|
||||
else:
|
||||
cur_input_ptr = input_ptr + slice_id * input_d0_stride
|
||||
if use_fp8_w8a8:
|
||||
cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
|
||||
tl.pointer_type(tl.float8e4nv)
|
||||
)
|
||||
cur_b_scale_ptr = tl.load(b_scale_ptr + slice_id).to(
|
||||
tl.pointer_type(tl.float32)
|
||||
)
|
||||
else:
|
||||
cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
|
||||
tl.pointer_type(out_ptr.dtype.element_ty)
|
||||
)
|
||||
cur_b_scale_ptr = (
|
||||
tl.load(b_scale_ptr + slice_id).to(tl.pointer_type(tl.float32))
|
||||
if b_scale_ptr is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Identify the column indices of B to process.
|
||||
offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
|
||||
rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
|
||||
|
||||
# Identify A and B block pointers
|
||||
offset_k = tl.arange(0, BLOCK_K)
|
||||
a_ptr = (
|
||||
cur_input_ptr
|
||||
+ ram[:, None] * input_d1_stride
|
||||
+ offset_k[None, :] * input_d2_stride
|
||||
)
|
||||
b_ptr = (
|
||||
cur_lora_ptr
|
||||
+ cur_lora_d0_stride * lora_index
|
||||
+ offset_k[:, None] * cur_lora_d2_stride
|
||||
+ rbn[None, :] * cur_lora_d1_stride
|
||||
)
|
||||
|
||||
# Setup scale pointers for FP8/INT8 quantization
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise quantization - compute scale pointers for fp8_mm_k
|
||||
# a_scale: per-row base pointers, shape (BLOCK_M,)
|
||||
mm_a_scale_ptr = a_scale_ptr + ram * a_scale_m_stride
|
||||
|
||||
# b_scale: pre-compute N-dimension offset since fp8_mm_k doesn't know pid_n
|
||||
n_offset = pid_n * BLOCK_N
|
||||
offs_ns = (n_offset + tl.arange(0, BLOCK_N)) // group_n
|
||||
# Base pointer with lora offset + N-group offset baked in, shape (BLOCK_N,)
|
||||
mm_b_scale_ptr = (
|
||||
cur_b_scale_ptr
|
||||
+ lora_index * b_scale_l_stride
|
||||
+ offs_ns * b_scale_n_stride
|
||||
)
|
||||
elif per_channel_quant:
|
||||
# Per-channel for weights, shape (BLOCK_N,)
|
||||
b_scale_ptrs = (
|
||||
cur_b_scale_ptr + lora_index * b_scale_l_stride + rbn * b_scale_n_stride
|
||||
)
|
||||
b_scale = tl.load(b_scale_ptrs)
|
||||
# Per-token activation scale, only if a_scale_ptr provided
|
||||
a_scale = tl.load(a_scale_ptr + ram * a_scale_m_stride)[:, None]
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Tensor-wise quantization
|
||||
a_scale = tl.load(a_scale_ptr) if a_scale_ptr is not None else 1.0
|
||||
b_scale = tl.load(cur_b_scale_ptr + lora_index * b_scale_l_stride)
|
||||
# For non-block-wise, pass original pointers (not used in mm loop)
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
else:
|
||||
# Non-quantized path
|
||||
mm_a_scale_ptr = a_scale_ptr
|
||||
mm_b_scale_ptr = cur_b_scale_ptr
|
||||
|
||||
# Compute the block matrix product using fp8_mm_k
|
||||
# Note: For expand kernel, SPLIT_K=1, so we pass 1 for SPLIT_K
|
||||
accumulator = fp8_mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
mm_a_scale_ptr,
|
||||
mm_b_scale_ptr,
|
||||
input_d2_stride, # ak_stride
|
||||
cur_lora_d2_stride, # bk_stride
|
||||
a_scale_k_stride,
|
||||
b_scale_k_stride,
|
||||
offset_k,
|
||||
K,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
EVEN_K,
|
||||
1, # SPLIT_K = 1 for expand kernel
|
||||
group_k,
|
||||
group_n,
|
||||
use_fp8_w8a8,
|
||||
per_channel_quant,
|
||||
CAST_TYPE, # CAST_TYPE - cast FP8 B to A's dtype
|
||||
cur_lora_ptr.dtype.element_ty,
|
||||
USE_GDC,
|
||||
base_k=0,
|
||||
)
|
||||
|
||||
# Apply dequantization scales for non-block-wise quantization
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
pass # Already applied per block in fp8_mm_k
|
||||
else:
|
||||
# Tensor-wise or per-channel: apply scales after accumulation
|
||||
accumulator = accumulator * a_scale * b_scale
|
||||
|
||||
tiled_c = accumulator.to(out_ptr.dtype.element_ty)
|
||||
if SLICE_NUM == 1:
|
||||
cur_slice_start = slice_start_loc
|
||||
else:
|
||||
cur_slice_start = tl.load(slice_start_loc + slice_id)
|
||||
|
||||
# Identify the C output pointers to store the results of the accumulator.
|
||||
offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + cur_slice_start
|
||||
offset_cm = tl.arange(0, BLOCK_M)
|
||||
c_ptr = (
|
||||
out_ptr
|
||||
+ ram[:, None] * output_d0_stride
|
||||
+ offset_cn[None, :] * output_d1_stride
|
||||
)
|
||||
c_mask = (offset_cm[:, None] < M_LEN) & (offset_cn[None, :] < (cur_slice_start + N))
|
||||
|
||||
if ADD_INPUTS:
|
||||
tiled_out = tl.load(c_ptr, mask=c_mask)
|
||||
tiled_c += tiled_out
|
||||
tl.store(c_ptr, tiled_c, mask=c_mask)
|
||||
@@ -0,0 +1,403 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.lora.ops.triton_ops.fp8_kernel_utils import do_expand_kernel_fp8
|
||||
from vllm.lora.ops.triton_ops.utils import (
|
||||
_get_lora_b_ptr,
|
||||
get_lora_op_configs,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
_EXPAND_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
|
||||
|
||||
|
||||
def _get_expand_lora_scale_ptr(lora_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
`profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
"""
|
||||
key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights)
|
||||
|
||||
if (ptr_tensor := _EXPAND_LORA_SCALE_PTR_DICT.get(key)) is not None:
|
||||
return ptr_tensor
|
||||
|
||||
if len(lora_weights) > 1:
|
||||
tensor_ptrs = []
|
||||
for lora_weight in lora_weights:
|
||||
tensor_ptrs.append(lora_weight.data_ptr())
|
||||
ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64)
|
||||
else:
|
||||
# Single slice: return the actual tensor so the kernel can use it
|
||||
# directly without pointer indirection (matches SLICE_NUM == 1 path).
|
||||
ptr_tensor = lora_weights[0]
|
||||
|
||||
_EXPAND_LORA_SCALE_PTR_DICT[key] = ptr_tensor
|
||||
return _EXPAND_LORA_SCALE_PTR_DICT.get(key)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _lora_expand_kernel_fp8(
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
token_indices_sorted_by_lora_ids,
|
||||
num_tokens_per_lora,
|
||||
lora_token_start_loc,
|
||||
lora_ids,
|
||||
slice_start_loc,
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
input_d2_stride,
|
||||
ls_d0_ptr,
|
||||
ls_d1_ptr,
|
||||
ls_d2_ptr,
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
output_hs_ptr,
|
||||
group_n: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
ADD_INPUTS: tl.constexpr,
|
||||
CAST_TYPE: tl.constexpr,
|
||||
SLICE_NUM: tl.constexpr,
|
||||
SAME_STRIDE: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
FP8-compatible expand kernel wrapper.
|
||||
"""
|
||||
cta_n_num = tl.cdiv(N, BLOCK_N)
|
||||
cta_m_num = tl.cdiv(M, BLOCK_M)
|
||||
|
||||
pid_mn = tl.program_id(axis=0)
|
||||
pid_m = pid_mn % cta_m_num
|
||||
pid_n = (pid_mn // cta_m_num) % cta_n_num
|
||||
|
||||
slice_id = tl.program_id(axis=1)
|
||||
lora_idx = tl.program_id(axis=2)
|
||||
|
||||
lora_id = tl.load(lora_ids + lora_idx)
|
||||
if lora_id == -1:
|
||||
return
|
||||
|
||||
lora_m_size = tl.load(num_tokens_per_lora + lora_idx)
|
||||
|
||||
cta_m_offset = pid_m * BLOCK_M
|
||||
if cta_m_offset >= lora_m_size:
|
||||
return
|
||||
|
||||
curr_N = N if SAME_STRIDE else tl.load(output_hs_ptr + slice_id)
|
||||
if pid_n * BLOCK_N >= curr_N:
|
||||
return
|
||||
|
||||
cta_m_len = min(BLOCK_M, lora_m_size - cta_m_offset)
|
||||
|
||||
lora_m_indices_start = tl.load(lora_token_start_loc + lora_idx)
|
||||
cta_lora_seq_indices = (
|
||||
token_indices_sorted_by_lora_ids + lora_m_indices_start + cta_m_offset
|
||||
)
|
||||
|
||||
offset_m = tl.arange(0, BLOCK_M) % cta_m_len
|
||||
ram = tl.load(cta_lora_seq_indices + offset_m)
|
||||
|
||||
do_expand_kernel_fp8(
|
||||
pid_n,
|
||||
lora_id,
|
||||
slice_id,
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
curr_N,
|
||||
K,
|
||||
cta_m_len,
|
||||
ram,
|
||||
slice_start_loc,
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
input_d2_stride,
|
||||
ls_d0_ptr,
|
||||
ls_d1_ptr,
|
||||
ls_d2_ptr,
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
group_n,
|
||||
group_k,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
SAME_STRIDE,
|
||||
SLICE_NUM,
|
||||
EVEN_K,
|
||||
CAST_TYPE,
|
||||
ADD_INPUTS,
|
||||
USE_GDC,
|
||||
use_fp8_w8a8,
|
||||
per_channel_quant,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _lora_expand_fp8(
|
||||
inputs: torch.Tensor, # shape [num_slices, num_tokens, lora_rank]
|
||||
lora_b_weights: list[torch.Tensor], # FP8 [num_lora, hidden_size, lora_rank]
|
||||
output_tensor: torch.Tensor, # shape [num_tokens, hidden_size * num_slices]
|
||||
token_lora_mapping: torch.Tensor,
|
||||
token_indices_sorted_by_lora_ids: torch.Tensor,
|
||||
num_tokens_per_lora: torch.Tensor,
|
||||
lora_token_start_loc: torch.Tensor,
|
||||
lora_ids: torch.Tensor,
|
||||
no_lora_flag_cpu: torch.Tensor, # shape [1]
|
||||
num_active_loras: int, # number of active LoRAs (unused here, for API compat)
|
||||
b_scale: list[torch.Tensor], # LoRA B weight scale per slice
|
||||
a_scale: torch.Tensor | None = None, # Scale for shrink output (optional)
|
||||
offset_start: int = 0,
|
||||
add_inputs: bool = False,
|
||||
group_k: int = 0,
|
||||
group_n: int = 0,
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
FP8-compatible LoRA expand operation.
|
||||
|
||||
Args:
|
||||
inputs: Input tensor from shrink operation [num_slices, num_tokens, lora_rank]
|
||||
lora_b_weights: List of FP8 LoRA B weights per slice
|
||||
output_tensor: Output tensor
|
||||
a_scale: Optional scale for input (if input is quantized)
|
||||
b_scale: Weight quantization scales per slice
|
||||
token_lora_mapping: Token to LoRA ID mapping
|
||||
token_indices_sorted_by_lora_ids: Sorted token indices
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
offset_start (int, optional): Offset start for output_tensor.
|
||||
Defaults to 0.
|
||||
add_inputs (bool, optional): Whether to add the input tensor to the
|
||||
output tensor. Defaults to False.
|
||||
group_k (int, optional): Block size for K in block-wise quantization.
|
||||
group_n (int, optional): Block size for N in block-wise quantization.
|
||||
use_fp8_w8a8 (bool, optional): Whether to use FP8 W8A8 quantization.
|
||||
per_channel_quant (bool, optional): Whether to use per-channel quantization.
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
# None of the inputs require LoRA.
|
||||
return
|
||||
|
||||
if use_fp8_w8a8:
|
||||
assert inputs.dtype in [
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e5m2,
|
||||
]
|
||||
for weight in lora_b_weights:
|
||||
assert weight.dtype in [
|
||||
torch.float8_e5m2,
|
||||
torch.float8_e4m3fn,
|
||||
]
|
||||
else:
|
||||
assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
|
||||
for weight in lora_b_weights:
|
||||
assert weight.dtype in [torch.float16, torch.bfloat16]
|
||||
assert inputs.size(0) == len(lora_b_weights)
|
||||
assert output_tensor.is_contiguous()
|
||||
|
||||
# metadata sanity check.
|
||||
M = inputs.size(1)
|
||||
assert token_lora_mapping.size(0) == M
|
||||
assert token_lora_mapping.size(0) == token_indices_sorted_by_lora_ids.size(0)
|
||||
assert lora_ids.size(0) == num_tokens_per_lora.size(0)
|
||||
assert lora_token_start_loc.size(0) == lora_ids.size(0) + 1
|
||||
|
||||
(
|
||||
slice_start_tensor,
|
||||
lora_ptr_tensor,
|
||||
lora_strides_d0_tensor,
|
||||
lora_strides_d1_tensor,
|
||||
lora_strides_d2_tensor,
|
||||
hidden_sizes_tensor,
|
||||
same_stride,
|
||||
MAX_N,
|
||||
) = _get_lora_b_ptr(lora_b_weights, offset_start, inputs.device)
|
||||
|
||||
# Get scale pointers
|
||||
if b_scale is not None:
|
||||
b_scale_ptr_tensor = _get_expand_lora_scale_ptr(b_scale, inputs.device)
|
||||
else:
|
||||
b_scale_ptr_tensor = None
|
||||
K = lora_b_weights[0].shape[-1]
|
||||
ADD_INPUTS = add_inputs
|
||||
MAX_LORAS = lora_ids.size(0)
|
||||
|
||||
CAST_TYPE = False
|
||||
NUM_SLICES = len(lora_b_weights)
|
||||
|
||||
# Triton kernel configs.
|
||||
kernel_config = get_lora_op_configs(
|
||||
op_type="expand",
|
||||
max_loras=MAX_LORAS,
|
||||
batch=M,
|
||||
hidden_size=MAX_N,
|
||||
rank=K,
|
||||
num_slices=NUM_SLICES,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
BLOCK_M = kernel_config["block_m"]
|
||||
BLOCK_N = kernel_config["block_n"]
|
||||
BLOCK_K = kernel_config["block_k"]
|
||||
NUM_WARPS = kernel_config["num_warps"]
|
||||
NUM_CTAS = kernel_config.get("num_ctas", 1)
|
||||
NUM_STAGES = kernel_config["num_stages"]
|
||||
|
||||
EVEN_K = K % BLOCK_K == 0
|
||||
|
||||
grid = (
|
||||
triton.cdiv(M, BLOCK_M) * triton.cdiv(MAX_N, BLOCK_N),
|
||||
NUM_SLICES,
|
||||
num_active_loras,
|
||||
)
|
||||
# We disable PDL temporarily because LoRA kernels are not launching back-to-back,
|
||||
# making PDL invalid and affecting the kernel performance.
|
||||
use_gdc = False # supports_pdl(inputs.device)
|
||||
# Get scale strides
|
||||
if a_scale is not None:
|
||||
a_scale_m_stride = a_scale.stride(0) if a_scale.dim() > 1 else 0
|
||||
a_scale_k_stride = a_scale.stride(-1) if a_scale.dim() > 1 else 0
|
||||
else:
|
||||
a_scale_m_stride = 0
|
||||
a_scale_k_stride = 0
|
||||
|
||||
if b_scale is not None and b_scale[0].dim() > 0:
|
||||
b_scale_l_stride = b_scale[0].stride(0) if b_scale[0].dim() > 0 else 0
|
||||
b_scale_n_stride = (
|
||||
b_scale[0].stride(-2)
|
||||
if b_scale[0].dim() > 2
|
||||
else (b_scale[0].stride(-1) if b_scale[0].dim() > 1 else 1)
|
||||
)
|
||||
b_scale_k_stride = b_scale[0].stride(-1) if b_scale[0].dim() > 2 else 0
|
||||
else:
|
||||
b_scale_l_stride = 1
|
||||
b_scale_n_stride = 0
|
||||
b_scale_k_stride = 0
|
||||
|
||||
_lora_expand_kernel_fp8[grid](
|
||||
inputs,
|
||||
lora_ptr_tensor,
|
||||
output_tensor,
|
||||
a_scale,
|
||||
b_scale_ptr_tensor,
|
||||
M,
|
||||
MAX_N,
|
||||
K,
|
||||
token_indices_sorted_by_lora_ids,
|
||||
num_tokens_per_lora,
|
||||
lora_token_start_loc,
|
||||
lora_ids,
|
||||
slice_start_tensor,
|
||||
inputs.stride(0),
|
||||
inputs.stride(1),
|
||||
inputs.stride(2),
|
||||
lora_strides_d0_tensor,
|
||||
lora_strides_d1_tensor,
|
||||
lora_strides_d2_tensor,
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
output_tensor.stride(0),
|
||||
output_tensor.stride(1),
|
||||
hidden_sizes_tensor,
|
||||
group_n,
|
||||
group_k,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
EVEN_K,
|
||||
ADD_INPUTS,
|
||||
CAST_TYPE,
|
||||
NUM_SLICES,
|
||||
same_stride,
|
||||
use_gdc,
|
||||
use_fp8_w8a8=use_fp8_w8a8,
|
||||
per_channel_quant=per_channel_quant,
|
||||
num_warps=NUM_WARPS,
|
||||
num_ctas=NUM_CTAS,
|
||||
num_stages=NUM_STAGES,
|
||||
launch_pdl=use_gdc,
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def _lora_expand_fp8_fake(
|
||||
inputs: torch.Tensor,
|
||||
lora_b_weights: list[torch.Tensor],
|
||||
output_tensor: torch.Tensor,
|
||||
token_lora_mapping: torch.Tensor,
|
||||
token_indices_sorted_by_lora_ids: torch.Tensor,
|
||||
num_tokens_per_lora: torch.Tensor,
|
||||
lora_token_start_loc: torch.Tensor,
|
||||
lora_ids: torch.Tensor,
|
||||
no_lora_flag_cpu: torch.Tensor,
|
||||
num_active_loras: int,
|
||||
b_scale: list[torch.Tensor],
|
||||
a_scale: torch.Tensor | None = None,
|
||||
offset_start: int = 0,
|
||||
add_inputs: bool = False,
|
||||
group_k: int = 0,
|
||||
group_n: int = 0,
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
direct_register_custom_op(
|
||||
op_name="lora_expand_fp8",
|
||||
op_func=_lora_expand_fp8,
|
||||
mutates_args=["output_tensor"],
|
||||
fake_impl=_lora_expand_fp8_fake,
|
||||
)
|
||||
lora_expand_fp8 = torch.ops.vllm.lora_expand_fp8
|
||||
|
||||
except AttributeError:
|
||||
lora_expand_fp8 = _lora_expand_fp8
|
||||
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.lora.ops.triton_ops.fp8_kernel_utils import do_shrink_kernel_fp8
|
||||
from vllm.lora.ops.triton_ops.utils import _get_lora_a_ptr, get_lora_op_configs
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
_SHRINK_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], tuple] = {}
|
||||
|
||||
|
||||
def _get_shrink_lora_scale_ptr(
|
||||
lora_scale_weights: list[torch.Tensor], device: torch.device
|
||||
):
|
||||
"""
|
||||
`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
`profile_run`. After this, it remains constant and subsequent usage is
|
||||
through LUT.
|
||||
|
||||
Returns a tuple of (scale_ptr_tensor, l_stride, n_stride, k_stride).
|
||||
|
||||
Supports scale tensors of varying dimensionality:
|
||||
- 1D: (lora_num,) — tensor-wise quantization
|
||||
- 2D: (lora_num, N) — per-channel quantization
|
||||
- 3D: (lora_num, N, K) — block-wise quantization
|
||||
- 4D: (lora_num, 1, N, K) — block-wise with extra dim (squeezed to 3D)
|
||||
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
"""
|
||||
key = tuple(lora_weight.data_ptr() for lora_weight in lora_scale_weights)
|
||||
|
||||
if values := _SHRINK_LORA_SCALE_PTR_DICT.get(key):
|
||||
return values
|
||||
|
||||
tensor_ptrs = []
|
||||
scale_l_strides = []
|
||||
scale_n_strides = []
|
||||
scale_k_strides = []
|
||||
for lora_scale_weight in lora_scale_weights:
|
||||
if lora_scale_weight.ndim == 4: # shape:(lora_num,1,size,rank)
|
||||
assert lora_scale_weight.size(1) == 1
|
||||
lora_scale_weight = lora_scale_weight.squeeze(dim=1)
|
||||
assert 1 <= lora_scale_weight.ndim <= 3
|
||||
assert lora_scale_weight.is_contiguous()
|
||||
tensor_ptrs.append(lora_scale_weight.data_ptr())
|
||||
scale_l_strides.append(
|
||||
lora_scale_weight.stride(0) if lora_scale_weight.ndim > 0 else 0
|
||||
)
|
||||
scale_n_strides.append(
|
||||
lora_scale_weight.stride(-2)
|
||||
if lora_scale_weight.ndim > 2
|
||||
else (lora_scale_weight.stride(-1) if lora_scale_weight.ndim > 1 else 1)
|
||||
)
|
||||
scale_k_strides.append(
|
||||
lora_scale_weight.stride(-1) if lora_scale_weight.ndim > 2 else 0
|
||||
)
|
||||
if len(lora_scale_weights) > 1:
|
||||
scale_ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64)
|
||||
else:
|
||||
scale_ptr_tensor = lora_scale_weights[0]
|
||||
|
||||
if (
|
||||
len(set(scale_l_strides)) > 1
|
||||
or len(set(scale_n_strides)) > 1
|
||||
or len(set(scale_k_strides)) > 1
|
||||
):
|
||||
raise ValueError("All LoRA scale weights must have the same stride.")
|
||||
|
||||
_SHRINK_LORA_SCALE_PTR_DICT[key] = (
|
||||
scale_ptr_tensor,
|
||||
scale_l_strides[0],
|
||||
scale_n_strides[0],
|
||||
scale_k_strides[0],
|
||||
)
|
||||
return _SHRINK_LORA_SCALE_PTR_DICT.get(key)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _lora_shrink_kernel_fp8(
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
token_indices_sorted_by_lora_ids,
|
||||
num_tokens_per_lora,
|
||||
lora_token_start_loc,
|
||||
lora_ids,
|
||||
scaling,
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
lora_d0_stride,
|
||||
lora_d1_stride,
|
||||
lora_d2_stride,
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
output_d2_stride,
|
||||
group_n: tl.constexpr,
|
||||
group_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
GROUP_SIZE_M: tl.constexpr,
|
||||
SLICE_NUM: tl.constexpr,
|
||||
USE_GDC: tl.constexpr, ## should always be false in shrink kernel
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
cta_n_num = tl.cdiv(N, BLOCK_N)
|
||||
cta_m_num = tl.cdiv(M, BLOCK_M)
|
||||
|
||||
pid_sk_m_n = tl.program_id(axis=0)
|
||||
pid_sk = pid_sk_m_n % SPLIT_K
|
||||
|
||||
pid_m_n = pid_sk_m_n // SPLIT_K
|
||||
num_pid_in_group = GROUP_SIZE_M * cta_n_num
|
||||
group_id = pid_m_n // num_pid_in_group
|
||||
first_pid_m = group_id * GROUP_SIZE_M
|
||||
|
||||
group_size_m = min(cta_m_num - first_pid_m, GROUP_SIZE_M)
|
||||
|
||||
# Column-major ordering within groups for better cache reuse
|
||||
pid_m = first_pid_m + ((pid_m_n % num_pid_in_group) % group_size_m)
|
||||
pid_n = (pid_m_n % num_pid_in_group) // group_size_m
|
||||
|
||||
slice_id = tl.program_id(axis=1)
|
||||
lora_idx = tl.program_id(axis=2)
|
||||
|
||||
lora_id = tl.load(lora_ids + lora_idx)
|
||||
if lora_id == -1:
|
||||
# Early exit for the no-lora case.
|
||||
return
|
||||
|
||||
lora_m_size = tl.load(num_tokens_per_lora + lora_idx)
|
||||
|
||||
cta_m_offset = pid_m * BLOCK_M
|
||||
if cta_m_offset >= lora_m_size:
|
||||
# Early exit CTA.
|
||||
return
|
||||
|
||||
# num rows this CTA should process.
|
||||
cta_m_len = min(BLOCK_M, lora_m_size - cta_m_offset)
|
||||
|
||||
# Identify all rows that this CTA should process.
|
||||
lora_m_indices_start = tl.load(lora_token_start_loc + lora_idx)
|
||||
cta_lora_seq_indices = (
|
||||
token_indices_sorted_by_lora_ids + lora_m_indices_start + cta_m_offset
|
||||
)
|
||||
|
||||
# Load all relevant row indices.
|
||||
offset_m = tl.arange(0, BLOCK_M) % cta_m_len
|
||||
ram = tl.load(cta_lora_seq_indices + offset_m)
|
||||
|
||||
do_shrink_kernel_fp8(
|
||||
pid_n,
|
||||
pid_sk,
|
||||
slice_id,
|
||||
lora_id,
|
||||
input_ptr,
|
||||
lora_ptr,
|
||||
out_ptr,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr,
|
||||
N,
|
||||
K,
|
||||
cta_m_len,
|
||||
ram, # array identifying the rows of Input ptr to operate on
|
||||
# input strides
|
||||
input_d0_stride,
|
||||
input_d1_stride,
|
||||
# lora strides
|
||||
lora_d0_stride,
|
||||
lora_d1_stride,
|
||||
lora_d2_stride,
|
||||
# scale strides
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
# output strides
|
||||
output_d0_stride,
|
||||
output_d1_stride,
|
||||
output_d2_stride,
|
||||
scaling,
|
||||
# block size for block-wise quantization
|
||||
group_n,
|
||||
group_k,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
EVEN_K,
|
||||
SPLIT_K,
|
||||
SLICE_NUM,
|
||||
USE_GDC,
|
||||
use_fp8_w8a8,
|
||||
per_channel_quant,
|
||||
launch_pdl,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _lora_shrink_fp8(
|
||||
inputs: torch.Tensor, # shape [num_tokens, hidden_size] - FP8 or FP16/BF16
|
||||
lora_a_weights: list[
|
||||
torch.Tensor
|
||||
], # shape [num_loras, lora_rank, hidden_size] - FP8 or FP16/BF16
|
||||
output_tensor: torch.Tensor, # shape [num_slices, num_tokens, lora_rank]
|
||||
token_lora_mapping: torch.Tensor, # shape [num_tokens]
|
||||
token_indices_sorted_by_lora_ids: torch.Tensor, # shape [num_tokens]
|
||||
num_tokens_per_lora: torch.Tensor, # shape [max-loras + 1]
|
||||
lora_token_start_loc: torch.Tensor, # shape [max-loras + 2]
|
||||
lora_ids: torch.Tensor, # shape [max-loras + 1]
|
||||
no_lora_flag_cpu: torch.Tensor, # shape [1]
|
||||
num_active_loras: int, # number of active LoRAs (unused here, for API compat)
|
||||
scaling: float,
|
||||
b_scale: list[torch.Tensor], # LoRA weight scale per slice
|
||||
a_scale: torch.Tensor | None = None, # Activation scale - per-token or block-wise
|
||||
group_k: int = 0, # Block size for K in block-wise quantization (0 = tensor-wise)
|
||||
group_n: int = 0, # Block size for N in block-wise quantization
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
|
||||
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
|
||||
output_tensor: Output tensor (FP16/BF16/FP32)
|
||||
token_lora_mapping: Token to LoRA ID mapping
|
||||
token_indices_sorted_by_lora_ids: Sorted token indices
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
scaling: LoRA scaling factor
|
||||
a_scale: Activation quantization scales
|
||||
b_scale: Weight quantization scales per slice
|
||||
group_k: Block size for K dimension quantization
|
||||
group_n: Block size for N dimension quantization
|
||||
use_fp8_w8a8: Whether to use FP8 weights and activations
|
||||
per_channel_quant: Whether to use per-channel quantization
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
# None of the inputs require LoRA.
|
||||
return
|
||||
|
||||
assert inputs.size(1) == lora_a_weights[0].size(-1)
|
||||
assert inputs.is_contiguous()
|
||||
assert output_tensor.is_contiguous()
|
||||
|
||||
# metadata sanity check
|
||||
M = inputs.size(0)
|
||||
assert token_lora_mapping.size(0) == M
|
||||
assert token_lora_mapping.size(0) == token_indices_sorted_by_lora_ids.size(0)
|
||||
assert lora_ids.size(0) == num_tokens_per_lora.size(0)
|
||||
assert lora_token_start_loc.size(0) == lora_ids.size(0) + 1
|
||||
|
||||
output_tensor.zero_()
|
||||
|
||||
# Get LoRA weight pointers
|
||||
(lora_ptr_tensor, lora_strides_d0, lora_strides_d1, lora_strides_d2) = (
|
||||
_get_lora_a_ptr(lora_a_weights, inputs.device)
|
||||
)
|
||||
|
||||
# Get scale pointers if using FP8
|
||||
if use_fp8_w8a8:
|
||||
assert a_scale is not None, "a_scale required for FP8 w8a8"
|
||||
assert b_scale is not None, "b_scale required for FP8"
|
||||
|
||||
b_scale_ptr_tensor, b_scale_l_stride, b_scale_n_stride, b_scale_k_stride = (
|
||||
_get_shrink_lora_scale_ptr(b_scale, inputs.device)
|
||||
)
|
||||
a_scale_ptr = (
|
||||
a_scale if a_scale is not None else torch.tensor(1.0, device=inputs.device)
|
||||
)
|
||||
else:
|
||||
b_scale_ptr_tensor = torch.tensor(0, device=inputs.device)
|
||||
b_scale_l_stride = 0
|
||||
b_scale_n_stride = 0
|
||||
b_scale_k_stride = 0
|
||||
a_scale_ptr = torch.tensor(0, device=inputs.device)
|
||||
|
||||
N, K = lora_a_weights[0].shape[-2:] # K=hidden_size, N=rank
|
||||
NUM_SLICES = len(lora_a_weights)
|
||||
MAX_LORAS = lora_ids.size(0)
|
||||
|
||||
# Triton kernel configs
|
||||
kernel_config = get_lora_op_configs(
|
||||
"shrink",
|
||||
max_loras=MAX_LORAS,
|
||||
batch=M,
|
||||
hidden_size=K,
|
||||
rank=N,
|
||||
num_slices=NUM_SLICES,
|
||||
)
|
||||
BLOCK_M = kernel_config["block_m"]
|
||||
BLOCK_N = kernel_config["block_n"]
|
||||
BLOCK_K = kernel_config["block_k"]
|
||||
SPLIT_K = kernel_config["split_k"]
|
||||
NUM_WARPS = kernel_config["num_warps"]
|
||||
NUM_STAGES = kernel_config["num_stages"]
|
||||
NUM_CTAS = kernel_config["num_ctas"]
|
||||
GROUP_SIZE_M = kernel_config.get("group_size_m", 8)
|
||||
assert BLOCK_K is not None and SPLIT_K is not None
|
||||
EVEN_K = K % (BLOCK_K * SPLIT_K) == 0
|
||||
|
||||
# Grid configuration with column-major ordering support
|
||||
grid = (
|
||||
SPLIT_K * triton.cdiv(M, BLOCK_M) * triton.cdiv(N, BLOCK_N),
|
||||
NUM_SLICES,
|
||||
num_active_loras,
|
||||
)
|
||||
|
||||
# Determine scale strides
|
||||
if use_fp8_w8a8:
|
||||
if a_scale is not None and a_scale.ndim == 2:
|
||||
a_scale_m_stride = a_scale.stride(0)
|
||||
a_scale_k_stride = a_scale.stride(1)
|
||||
else:
|
||||
a_scale_m_stride = 0
|
||||
a_scale_k_stride = 0
|
||||
else:
|
||||
a_scale_m_stride = 0
|
||||
a_scale_k_stride = 0
|
||||
|
||||
# We disable PDL temporarily because LoRA kernels are not launching back-to-back,
|
||||
# making PDL invalid and affecting the kernel performance.
|
||||
use_gdc = False # supports_pdl(inputs.device)
|
||||
_lora_shrink_kernel_fp8[grid](
|
||||
inputs,
|
||||
lora_ptr_tensor,
|
||||
output_tensor,
|
||||
a_scale_ptr,
|
||||
b_scale_ptr_tensor,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
token_indices_sorted_by_lora_ids,
|
||||
num_tokens_per_lora,
|
||||
lora_token_start_loc,
|
||||
lora_ids,
|
||||
scaling,
|
||||
inputs.stride(0),
|
||||
inputs.stride(1),
|
||||
lora_strides_d0,
|
||||
lora_strides_d1,
|
||||
lora_strides_d2,
|
||||
a_scale_m_stride,
|
||||
a_scale_k_stride,
|
||||
b_scale_l_stride,
|
||||
b_scale_n_stride,
|
||||
b_scale_k_stride,
|
||||
output_tensor.stride(0),
|
||||
output_tensor.stride(1),
|
||||
output_tensor.stride(2),
|
||||
group_n,
|
||||
group_k,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
EVEN_K,
|
||||
SPLIT_K,
|
||||
GROUP_SIZE_M,
|
||||
NUM_SLICES,
|
||||
use_gdc,
|
||||
use_fp8_w8a8,
|
||||
per_channel_quant,
|
||||
use_gdc,
|
||||
num_warps=NUM_WARPS,
|
||||
num_ctas=NUM_CTAS,
|
||||
num_stages=NUM_STAGES,
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def _lora_shrink_fp8_fake(
|
||||
inputs: torch.Tensor,
|
||||
lora_a_weights: list[torch.Tensor],
|
||||
output_tensor: torch.Tensor,
|
||||
token_lora_mapping: torch.Tensor,
|
||||
token_indices_sorted_by_lora_ids: torch.Tensor,
|
||||
num_tokens_per_lora: torch.Tensor,
|
||||
lora_token_start_loc: torch.Tensor,
|
||||
lora_ids: torch.Tensor,
|
||||
no_lora_flag_cpu: torch.Tensor,
|
||||
num_active_loras: int,
|
||||
scaling: float,
|
||||
b_scale: list[torch.Tensor], # LoRA weight scale per slice
|
||||
a_scale: torch.Tensor | None = None, # Activation scale - per-token or block-wise
|
||||
group_k: int = 0, # Block size for K in block-wise quantization (0 = tensor-wise)
|
||||
group_n: int = 0, # Block size for N in block-wise quantization
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
direct_register_custom_op(
|
||||
op_name="lora_shrink_fp8",
|
||||
op_func=_lora_shrink_fp8,
|
||||
mutates_args=["output_tensor"],
|
||||
fake_impl=_lora_shrink_fp8_fake,
|
||||
)
|
||||
lora_shrink_fp8 = torch.ops.vllm.lora_shrink_fp8
|
||||
|
||||
except AttributeError:
|
||||
lora_shrink_fp8 = _lora_shrink_fp8
|
||||
@@ -252,7 +252,7 @@ def get_lora_op_configs(
|
||||
default = {
|
||||
"block_m": 64,
|
||||
"block_n": 64 if num_slices > 1 else 128,
|
||||
"block_k": 16,
|
||||
"block_k": 32,
|
||||
"num_warps": 4,
|
||||
"num_ctas": 1,
|
||||
"num_stages": 2,
|
||||
|
||||
Reference in New Issue
Block a user