[xpu][lora]: Align LoRA implementation with Punica GPU: fix _apply_expand rank mismatch, add_inputs hardcode, and MoE EP (#45368)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
@@ -81,7 +81,7 @@ steps:
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
set -o pipefail &&
|
||||
pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]"'
|
||||
pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]" --deselect="tests/lora/test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-32000-32-4-4]" --deselect="tests/lora/test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-32000-32-4-4]"'
|
||||
|
||||
- label: LoRA Punica FP8/XPU Ops
|
||||
timeout_in_minutes: 45
|
||||
|
||||
@@ -3,29 +3,27 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def bgmv_shrink(
|
||||
def _bgmv_shrink_impl(
|
||||
inputs: torch.Tensor,
|
||||
lora_a_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
scaling: float = 1.0,
|
||||
scaling: float,
|
||||
) -> None:
|
||||
torch.ops._xpu_C.bgmv_shrink(
|
||||
output_tensor, inputs, lora_a_weights, lora_indices_tensor, scaling
|
||||
)
|
||||
|
||||
|
||||
def bgmv_expand(
|
||||
def _bgmv_expand_impl(
|
||||
inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
add_inputs: bool = True,
|
||||
add_inputs: bool,
|
||||
) -> None:
|
||||
weight_out_dim = lora_b_weights.size(-2)
|
||||
output_dim = output_tensor.size(1)
|
||||
@@ -65,14 +63,14 @@ def bgmv_expand(
|
||||
)
|
||||
|
||||
|
||||
def bgmv_expand_slice(
|
||||
def _bgmv_expand_slice_impl(
|
||||
inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
slice_offset: int,
|
||||
slice_size: int,
|
||||
add_inputs: bool = True,
|
||||
add_inputs: bool,
|
||||
) -> None:
|
||||
assert slice_size == lora_b_weights.size(-2)
|
||||
assert slice_offset + slice_size <= output_tensor.size(1)
|
||||
@@ -85,3 +83,101 @@ def bgmv_expand_slice(
|
||||
slice_size,
|
||||
add_inputs,
|
||||
)
|
||||
|
||||
|
||||
def _bgmv_shrink_fake(
|
||||
inputs: torch.Tensor,
|
||||
lora_a_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
scaling: float,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _bgmv_expand_fake(
|
||||
inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
add_inputs: bool,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _bgmv_expand_slice_fake(
|
||||
inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
slice_offset: int,
|
||||
slice_size: int,
|
||||
add_inputs: bool,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="xpu_bgmv_shrink",
|
||||
op_func=_bgmv_shrink_impl,
|
||||
mutates_args=["output_tensor"],
|
||||
fake_impl=_bgmv_shrink_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="xpu_bgmv_expand",
|
||||
op_func=_bgmv_expand_impl,
|
||||
mutates_args=["output_tensor"],
|
||||
fake_impl=_bgmv_expand_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="xpu_bgmv_expand_slice",
|
||||
op_func=_bgmv_expand_slice_impl,
|
||||
mutates_args=["output_tensor"],
|
||||
fake_impl=_bgmv_expand_slice_fake,
|
||||
)
|
||||
|
||||
|
||||
def bgmv_shrink(
|
||||
inputs: torch.Tensor,
|
||||
lora_a_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
scaling: float = 1.0,
|
||||
) -> None:
|
||||
torch.ops.vllm.xpu_bgmv_shrink(
|
||||
inputs, lora_a_weights, output_tensor, lora_indices_tensor, scaling
|
||||
)
|
||||
|
||||
|
||||
def bgmv_expand(
|
||||
inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
add_inputs: bool = True,
|
||||
) -> None:
|
||||
torch.ops.vllm.xpu_bgmv_expand(
|
||||
inputs, lora_b_weights, output_tensor, lora_indices_tensor, add_inputs
|
||||
)
|
||||
|
||||
|
||||
def bgmv_expand_slice(
|
||||
inputs: torch.Tensor,
|
||||
lora_b_weights: torch.Tensor,
|
||||
output_tensor: torch.Tensor,
|
||||
lora_indices_tensor: torch.Tensor,
|
||||
slice_offset: int,
|
||||
slice_size: int,
|
||||
add_inputs: bool = True,
|
||||
) -> None:
|
||||
torch.ops.vllm.xpu_bgmv_expand_slice(
|
||||
inputs,
|
||||
lora_b_weights,
|
||||
output_tensor,
|
||||
lora_indices_tensor,
|
||||
slice_offset,
|
||||
slice_size,
|
||||
add_inputs,
|
||||
)
|
||||
|
||||
@@ -62,6 +62,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
captured_lora_counts=captured_lora_counts,
|
||||
)
|
||||
|
||||
# When speculative decoding is enabled, max_num_samples is
|
||||
# max_batches * (num_speculative_decoding_tokens + 1).
|
||||
# This line can be optimized by replacing max_num_batched_tokens
|
||||
# to max_batches * (num_speculative_decoding_tokens + 1).
|
||||
self.prompt_mapping_meta = LoRAKernelMeta.make(
|
||||
self.max_loras,
|
||||
max_num_batched_tokens,
|
||||
@@ -106,6 +110,14 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
add_inputs: bool,
|
||||
):
|
||||
token_lora_indices = self._get_token_lora_indices(x)
|
||||
# After tensor-parallel all-gather (non-fully-sharded LoRA), x may
|
||||
# have been gathered along the rank dim so x.size(1) == max_lora_rank
|
||||
# * tp_size, while lora_b only uses max_lora_rank elements. The XPU
|
||||
# C++ kernel requires inputs.size(1) == lora_b.size(-1), so truncate
|
||||
# to the actual rank. x[:, :rank] is non-contiguous, hence the copy.
|
||||
rank = w_t_all.size(-1)
|
||||
if x.size(1) != rank:
|
||||
x = x[:, :rank].contiguous()
|
||||
bgmv_expand_slice(
|
||||
x, w_t_all, y, token_lora_indices, y_offset, y_slice_size, add_inputs
|
||||
)
|
||||
@@ -179,7 +191,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
offset_start += output_slices[slice_idx]
|
||||
y.view_as(y_org)
|
||||
y = y.view_as(y_org)
|
||||
|
||||
def add_lora_embedding(
|
||||
self,
|
||||
@@ -227,7 +239,6 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
@ lora_b_stacked[indices[i], layer_idx, :, :]
|
||||
* scale
|
||||
).squeeze(0)
|
||||
|
||||
Args:
|
||||
y (torch.Tensor): Output tensor. Will be changed in-place.
|
||||
x (torch.Tensor): Input tensor
|
||||
@@ -240,13 +251,17 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
|
||||
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
|
||||
|
||||
if buffer is None:
|
||||
r = lora_b_stacked[0].size(-1)
|
||||
buffer = torch.zeros( # type: ignore
|
||||
(len(output_slices), x.size(0), r),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
assert buffer is None, (
|
||||
"To minimize overhead, the buffer should be created by "
|
||||
".add_lora_linear() instead of being passed in."
|
||||
)
|
||||
r = lora_b_stacked[0].size(-1)
|
||||
buffer = torch.zeros( # type: ignore
|
||||
(len(output_slices), x.size(0), r),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
add_inputs = kwargs.pop("add_inputs", True)
|
||||
self.add_shrink(
|
||||
buffer, # type: ignore
|
||||
x,
|
||||
@@ -259,7 +274,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
buffer, # type: ignore
|
||||
lora_b_stacked,
|
||||
output_slices,
|
||||
add_inputs=True,
|
||||
add_inputs=add_inputs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -300,12 +315,16 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
y = y.view(-1, y.shape[-1])
|
||||
x = x.view(-1, x.shape[-1])
|
||||
r = lora_b_stacked.size(-1)
|
||||
if buffer is None:
|
||||
buffer = torch.zeros((x.size(0), r), dtype=x.dtype, device=x.device)
|
||||
|
||||
assert buffer is None, (
|
||||
"To minimize overhead, the buffer should be created by "
|
||||
".add_lora_linear() instead of being passed in."
|
||||
)
|
||||
buffer = torch.zeros((x.size(0), r), dtype=x.dtype, device=x.device)
|
||||
sampler_indices = torch.narrow(self._sampler_indices, 0, 0, x.size(0))
|
||||
bgmv_shrink(x, lora_a_stacked, buffer, sampler_indices, scale)
|
||||
bgmv_expand(buffer, lora_b_stacked, y, sampler_indices, add_inputs=True)
|
||||
return y.view_as(y_org)
|
||||
y = y.view_as(y_org)
|
||||
|
||||
def moe_lora_align_block_size(
|
||||
self,
|
||||
@@ -318,33 +337,62 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
expert_map: torch.Tensor | None = None,
|
||||
pad_sorted_ids: bool = False,
|
||||
naive_block_assignment: bool = False,
|
||||
token_lora_mapping: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Aligns tokens and experts into block-sized chunks for LoRA-based
|
||||
mixture-of-experts (MoE) execution.
|
||||
|
||||
When `token_lora_mapping` is provided, it overrides the global mapping
|
||||
read from `self.token_mapping_meta`. This is how EP+LoRA injects the
|
||||
per-rank-local token→LoRA map after all-to-all dispatch.
|
||||
"""
|
||||
(token_lora_mapping, _, _, _, lora_ids, _, _) = (
|
||||
self.token_mapping_meta.meta_args(
|
||||
num_tokens, self.lora_config.specialize_active_lora
|
||||
)
|
||||
(
|
||||
token_lora_mapping_meta,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
lora_ids,
|
||||
_,
|
||||
_,
|
||||
) = self.token_mapping_meta.meta_args(
|
||||
num_tokens, self.lora_config.specialize_active_lora
|
||||
)
|
||||
if token_lora_mapping is None:
|
||||
token_lora_mapping = token_lora_mapping_meta
|
||||
# Under EP the caller passes local_num_experts but topk_ids carries
|
||||
# GLOBAL expert indices. The CUDA kernel uses num_experts to size
|
||||
# its bucketing table; with EP we must size by global_num_experts
|
||||
# so global topk_ids don't overflow. expert_map inside the kernel
|
||||
# then translates global→local so the output expert_ids are local
|
||||
# (mirrors the non-LoRA moe_align_block_size behavior).
|
||||
kernel_num_experts = (
|
||||
expert_map.numel() if expert_map is not None else num_experts
|
||||
)
|
||||
if naive_block_assignment:
|
||||
expert_ids = topk_ids.reshape(-1)
|
||||
sorted_ids = None
|
||||
num_tokens_post_pad = None
|
||||
else:
|
||||
max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
|
||||
max_num_tokens_padded = topk_ids.numel() + kernel_num_experts * (
|
||||
block_size - 1
|
||||
)
|
||||
if pad_sorted_ids:
|
||||
max_num_tokens_padded = round_up(max_num_tokens_padded, block_size)
|
||||
if topk_ids.numel() < kernel_num_experts:
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
sorted_ids = torch.empty(
|
||||
(max_loras * max_num_tokens_padded,),
|
||||
dtype=torch.int32,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
|
||||
# Expert ids must be set default to -1 to prevent a blank block
|
||||
expert_ids = torch.empty(
|
||||
# Expert ids are initialized to -1 so unused (lora, expert)
|
||||
# slots don't drive the LoRA Triton kernel into the wrong bucket.
|
||||
# The kernel overwrites only active slots.
|
||||
expert_ids = torch.full(
|
||||
(max_loras * max_num_m_blocks,),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
@@ -355,7 +403,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
ops.moe_lora_align_block_size(
|
||||
topk_ids,
|
||||
token_lora_mapping,
|
||||
num_experts,
|
||||
kernel_num_experts,
|
||||
block_size,
|
||||
max_loras,
|
||||
max_num_tokens_padded,
|
||||
@@ -365,11 +413,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
num_tokens_post_pad,
|
||||
adapter_enabled,
|
||||
lora_ids,
|
||||
expert_map,
|
||||
)
|
||||
if expert_map is not None:
|
||||
expert_ids = expert_map[expert_ids]
|
||||
|
||||
return None, sorted_ids, expert_ids, num_tokens_post_pad
|
||||
return token_lora_mapping, sorted_ids, expert_ids, num_tokens_post_pad
|
||||
|
||||
def add_lora_fused_moe(
|
||||
self,
|
||||
@@ -525,7 +572,8 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
|
||||
SPARSITY_FACTOR = 8
|
||||
naive_block_assignment = (
|
||||
expert_map is None
|
||||
not fully_sharded
|
||||
and expert_map is None
|
||||
and num_tokens * top_k * SPARSITY_FACTOR <= local_num_experts * max_loras
|
||||
)
|
||||
|
||||
@@ -543,6 +591,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
adapter_enabled,
|
||||
expert_map,
|
||||
naive_block_assignment=naive_block_assignment,
|
||||
token_lora_mapping=token_lora_mapping,
|
||||
)
|
||||
|
||||
_sorted = sorted_token_ids_lora
|
||||
@@ -567,6 +616,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
adapter_enabled,
|
||||
fully_sharded=fully_sharded,
|
||||
token_lora_mapping=token_lora_mapping,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
|
||||
return (
|
||||
@@ -680,4 +730,5 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
fully_sharded=fully_sharded,
|
||||
offset=offset,
|
||||
token_lora_mapping=token_lora_mapping,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user