[Hardware][TPU][V1] Multi-LoRA implementation for the V1 TPU backend (#14238)
Signed-off-by: Akshat Tripathi <akshat@krai.ai> Signed-off-by: Chengji Yao <chengjiyao@google.com> Co-authored-by: Chengji Yao <chengjiyao@google.com>
This commit is contained in:
co-authored by
Chengji Yao
parent
db593aa67f
commit
c20ef40fd0
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from vllm.lora.ops.xla_ops.lora_ops import (bgmv_expand, bgmv_expand_slice,
|
||||
bgmv_shrink)
|
||||
|
||||
__all__ = ["bgmv_expand", "bgmv_expand_slice", "bgmv_shrink"]
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import torch
|
||||
|
||||
# Required to register the custom ops
|
||||
import vllm.lora.ops.xla_ops.pallas # noqa # pylint: disable=unused-import
|
||||
|
||||
|
||||
def bgmv_expand(inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
add_inputs: bool = True):
|
||||
"""
|
||||
Args:
|
||||
inputs (torch.Tensor): Input tensor of shape [num_tokens, hidden_size].
|
||||
|
||||
lora_b_weights (torch.Tensor): LoRA weights of shape
|
||||
[num_loras, lora_rank, hidden_size].
|
||||
|
||||
output_tensor (torch.Tensor): output tensor of shape
|
||||
[num_tokens, hidden_size * num_slices].
|
||||
|
||||
lora_indices_tensor (torch.Tensor): Tensor of shape [num_tokens]
|
||||
indicating which LoRA matrix to use for each token.
|
||||
add_inputs (bool): Whether or not to add the input tensor to the output
|
||||
tensor.
|
||||
"""
|
||||
|
||||
outputs = torch.ops.xla.bgmv(inputs, lora_b_weights, lora_indices_tensor)
|
||||
n_tokens = outputs.size(0)
|
||||
|
||||
limit = output_tensor.shape[0]
|
||||
if outputs.shape[0] == 1 and output_tensor.shape[0] != 1:
|
||||
limit = 1
|
||||
|
||||
outputs = torch.cat(
|
||||
(outputs,
|
||||
torch.zeros((n_tokens, output_tensor.shape[1] - outputs.shape[1]),
|
||||
device=outputs.device)),
|
||||
dim=1)
|
||||
|
||||
if add_inputs:
|
||||
return output_tensor + outputs[:limit, :]
|
||||
else:
|
||||
return outputs[:limit, :]
|
||||
|
||||
|
||||
def bgmv_shrink(inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
scaling: float = 1.0):
|
||||
"""
|
||||
Args:
|
||||
inputs (torch.Tensor): Input tensor of shape [num_tokens, hidden_size].
|
||||
lora_b_weights (torch.Tensor): LoRA weights of shape
|
||||
[num_loras, lora_rank, hidden_size].
|
||||
output_tensor (torch.Tensor): (Unused) output tensor (placeholder).
|
||||
lora_indices_tensor (torch.Tensor): Tensor of shape [num_tokens]
|
||||
indicating which LoRA matrix to use for each token.
|
||||
scaling (float, optional): Scalar multiplier applied to the output.
|
||||
"""
|
||||
|
||||
return scaling * torch.ops.xla.bgmv(inputs, lora_b_weights,
|
||||
lora_indices_tensor)
|
||||
|
||||
|
||||
def bgmv_expand_slice(inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
slice_offset: int,
|
||||
slice_size: int,
|
||||
add_inputs: bool = True):
|
||||
"""
|
||||
Args:
|
||||
inputs (torch.Tensor): Input tensor of shape [num_tokens, hidden_size].
|
||||
|
||||
lora_b_weights (torch.Tensor): LoRA weights of shape
|
||||
[num_loras, lora_rank, hidden_size].
|
||||
|
||||
output_tensor (torch.Tensor): output tensor of shape
|
||||
[num_tokens, hidden_size * num_slices].
|
||||
|
||||
lora_indices_tensor (torch.Tensor): Tensor of shape [num_tokens]
|
||||
indicating which LoRA matrix to use for each token.
|
||||
add_inputs (bool): Whether or not to add the input tensor to the output
|
||||
tensor.
|
||||
"""
|
||||
outputs = torch.ops.xla.bgmv(inputs, lora_b_weights, lora_indices_tensor)
|
||||
n_tokens = outputs.size(0)
|
||||
|
||||
outputs = torch.cat((
|
||||
torch.zeros((n_tokens, slice_offset), device=outputs.device),
|
||||
outputs,
|
||||
torch.zeros(
|
||||
(n_tokens, output_tensor.shape[1] - (slice_offset + slice_size)),
|
||||
device=outputs.device),
|
||||
),
|
||||
dim=1)
|
||||
|
||||
if add_inputs:
|
||||
return output_tensor + outputs
|
||||
else:
|
||||
return outputs
|
||||
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import functools
|
||||
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
import torch
|
||||
from jax.experimental import pallas as pl
|
||||
from jax.experimental.pallas import tpu as pltpu
|
||||
from torch.library import impl
|
||||
from torch_xla.experimental.custom_kernel import (XLA_LIB, jax_import_guard,
|
||||
make_kernel_from_pallas)
|
||||
|
||||
# TODO: Tune these
|
||||
TOKENS_BLOCK = 16
|
||||
LORA_RANK_BLOCK = 128
|
||||
DIM_BLOCK_SIZE = 128
|
||||
|
||||
|
||||
def _bgmv_kernel(bT: int, bL: int, idx_ref, inp_ref, lora_ref, out_ref,
|
||||
acc_ref, mask_ref):
|
||||
|
||||
@pl.when(pl.program_id(2) == 0)
|
||||
def _():
|
||||
acc_ref[...] = jnp.zeros_like(acc_ref[...], dtype=jnp.float32)
|
||||
|
||||
t = pl.program_id(0)
|
||||
|
||||
for i in range(bT):
|
||||
idx = idx_ref[i + bT * t]
|
||||
mask_ref[...] = jnp.zeros_like(mask_ref[...], dtype=jnp.float32)
|
||||
mask_ref[i, :] = jnp.ones((bL, ), dtype=jnp.float32)
|
||||
|
||||
acc_ref[...] += jax.lax.dot_general(
|
||||
inp_ref[...],
|
||||
lora_ref[idx, ...], (((1, ), (1, )), ((), ())),
|
||||
preferred_element_type=jnp.float32) * mask_ref[...]
|
||||
|
||||
@pl.when(pl.program_id(2) == pl.num_programs(2) - 1)
|
||||
def _():
|
||||
out_ref[...] = acc_ref[...].astype(out_ref.dtype)
|
||||
|
||||
|
||||
@jax.jit
|
||||
def _bgmv(
|
||||
idxs: jax.Array, # (T, ) int32
|
||||
inputs: jax.Array, # (T, D) model dtype
|
||||
loras: jax.Array # (N, L, D) model dtype
|
||||
) -> jax.Array: # (T, L) model dtype
|
||||
T, D = inputs.shape
|
||||
N, L, _ = loras.shape
|
||||
|
||||
return pl.pallas_call(
|
||||
kernel=functools.partial(_bgmv_kernel, TOKENS_BLOCK, LORA_RANK_BLOCK),
|
||||
out_shape=jax.ShapeDtypeStruct((T, L), dtype=inputs.dtype),
|
||||
grid_spec=pltpu.PrefetchScalarGridSpec(
|
||||
num_scalar_prefetch=1,
|
||||
grid=(T // TOKENS_BLOCK, L // LORA_RANK_BLOCK,
|
||||
D // DIM_BLOCK_SIZE),
|
||||
in_specs=[
|
||||
pl.BlockSpec((TOKENS_BLOCK, DIM_BLOCK_SIZE),
|
||||
lambda i, j, k, block_idx: (i, k)),
|
||||
pl.BlockSpec((N, LORA_RANK_BLOCK, DIM_BLOCK_SIZE),
|
||||
lambda i, j, k, block_idx: (0, j, k)),
|
||||
],
|
||||
out_specs=pl.BlockSpec((TOKENS_BLOCK, LORA_RANK_BLOCK),
|
||||
lambda i, j, k, block_idx: (i, j)),
|
||||
scratch_shapes=[
|
||||
pltpu.VMEM((TOKENS_BLOCK, LORA_RANK_BLOCK), jnp.float32),
|
||||
pltpu.VMEM((TOKENS_BLOCK, LORA_RANK_BLOCK), jnp.float32)
|
||||
]),
|
||||
compiler_params=pltpu.TPUCompilerParams(
|
||||
dimension_semantics=("parallel", "parallel", "arbitrary")),
|
||||
name="bgmv")(idxs, inputs, loras)
|
||||
|
||||
|
||||
def bgmv_shape_function(idxs, inputs, loras):
|
||||
T, _ = inputs.shape
|
||||
_, L, _ = loras.shape
|
||||
|
||||
return [((T, L), inputs.dtype)]
|
||||
|
||||
|
||||
XLA_LIB.define("bgmv(Tensor inputs, Tensor loras, Tensor idxs) -> Tensor", )
|
||||
|
||||
|
||||
@impl(XLA_LIB, "bgmv", "XLA")
|
||||
def bgmv_xla(inputs: torch.Tensor, loras: torch.Tensor, idxs: torch.IntTensor):
|
||||
inputs = inputs.to(dtype=loras.dtype)
|
||||
|
||||
if len(loras.shape) == 4:
|
||||
loras = loras.squeeze(axis=1)
|
||||
|
||||
jax_import_guard()
|
||||
kernel = make_kernel_from_pallas(_bgmv, bgmv_shape_function)
|
||||
|
||||
T, _ = inputs.shape
|
||||
_, L, D = loras.shape
|
||||
|
||||
# Pad the loras' rank if it's too low. This is to allow it to fit in a TPU
|
||||
# register. This has to happen in pytorch, doing it in Jax will lead to NaNs
|
||||
L1 = L
|
||||
if LORA_RANK_BLOCK > L or L % LORA_RANK_BLOCK != 0:
|
||||
L1 = (L // LORA_RANK_BLOCK + 1) * LORA_RANK_BLOCK
|
||||
|
||||
D1 = D
|
||||
if DIM_BLOCK_SIZE > D or D % DIM_BLOCK_SIZE != 0:
|
||||
D1 = (D // DIM_BLOCK_SIZE + 1) * DIM_BLOCK_SIZE
|
||||
|
||||
T1 = T
|
||||
if TOKENS_BLOCK > T or T % TOKENS_BLOCK != 0:
|
||||
T1 = (T // TOKENS_BLOCK + 1) * TOKENS_BLOCK
|
||||
|
||||
if D1 != D or L1 != L:
|
||||
loras = torch.nn.functional.pad(loras, (0, D1 - D, 0, L1 - L, 0, 0))
|
||||
if D1 != D or T1 != T:
|
||||
inputs = torch.nn.functional.pad(inputs, (0, D1 - D, 0, T1 - T))
|
||||
if T1 != T:
|
||||
idxs = torch.nn.functional.pad(idxs, ((0, T1 - T)))
|
||||
|
||||
return kernel(idxs, inputs, loras)[:T, :L]
|
||||
|
||||
|
||||
@impl(XLA_LIB, "bgmv", "CompositeExplicitAutograd")
|
||||
def bgmv_non_xla(inputs: torch.Tensor, loras: torch.Tensor,
|
||||
idxs: torch.IntTensor):
|
||||
T, _ = inputs.shape
|
||||
|
||||
if len(loras.shape) == 4:
|
||||
loras = loras.squeeze(axis=1)
|
||||
|
||||
_, L, _ = loras.shape
|
||||
|
||||
return torch.empty((T, L), device=inputs.device)
|
||||
Reference in New Issue
Block a user