[xpu][lora]: Align LoRA implementation with Punica GPU: fix _apply_expand rank mismatch, add_inputs hardcode, and MoE EP (#45368)

Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
Chaojun Zhang
2026-07-01 22:14:04 +08:00
committed by GitHub
parent f1cf6b0086
commit 13c49f9845
3 changed files with 182 additions and 35 deletions
+1 -1
View File
@@ -81,7 +81,7 @@ steps:
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
set -o pipefail &&
pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]"'
pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]" --deselect="tests/lora/test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-32000-32-4-4]" --deselect="tests/lora/test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-32000-32-4-4]"'
- label: LoRA Punica FP8/XPU Ops
timeout_in_minutes: 45
+105 -9
View File
@@ -3,29 +3,27 @@
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
from vllm.utils.torch_utils import direct_register_custom_op
def bgmv_shrink(
def _bgmv_shrink_impl(
inputs: torch.Tensor,
lora_a_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
scaling: float = 1.0,
scaling: float,
) -> None:
torch.ops._xpu_C.bgmv_shrink(
output_tensor, inputs, lora_a_weights, lora_indices_tensor, scaling
)
def bgmv_expand(
def _bgmv_expand_impl(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
add_inputs: bool = True,
add_inputs: bool,
) -> None:
weight_out_dim = lora_b_weights.size(-2)
output_dim = output_tensor.size(1)
@@ -65,14 +63,14 @@ def bgmv_expand(
)
def bgmv_expand_slice(
def _bgmv_expand_slice_impl(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
slice_offset: int,
slice_size: int,
add_inputs: bool = True,
add_inputs: bool,
) -> None:
assert slice_size == lora_b_weights.size(-2)
assert slice_offset + slice_size <= output_tensor.size(1)
@@ -85,3 +83,101 @@ def bgmv_expand_slice(
slice_size,
add_inputs,
)
def _bgmv_shrink_fake(
inputs: torch.Tensor,
lora_a_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
scaling: float,
) -> None:
return None
def _bgmv_expand_fake(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
add_inputs: bool,
) -> None:
return None
def _bgmv_expand_slice_fake(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
slice_offset: int,
slice_size: int,
add_inputs: bool,
) -> None:
return None
direct_register_custom_op(
op_name="xpu_bgmv_shrink",
op_func=_bgmv_shrink_impl,
mutates_args=["output_tensor"],
fake_impl=_bgmv_shrink_fake,
)
direct_register_custom_op(
op_name="xpu_bgmv_expand",
op_func=_bgmv_expand_impl,
mutates_args=["output_tensor"],
fake_impl=_bgmv_expand_fake,
)
direct_register_custom_op(
op_name="xpu_bgmv_expand_slice",
op_func=_bgmv_expand_slice_impl,
mutates_args=["output_tensor"],
fake_impl=_bgmv_expand_slice_fake,
)
def bgmv_shrink(
inputs: torch.Tensor,
lora_a_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
scaling: float = 1.0,
) -> None:
torch.ops.vllm.xpu_bgmv_shrink(
inputs, lora_a_weights, output_tensor, lora_indices_tensor, scaling
)
def bgmv_expand(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
add_inputs: bool = True,
) -> None:
torch.ops.vllm.xpu_bgmv_expand(
inputs, lora_b_weights, output_tensor, lora_indices_tensor, add_inputs
)
def bgmv_expand_slice(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
slice_offset: int,
slice_size: int,
add_inputs: bool = True,
) -> None:
torch.ops.vllm.xpu_bgmv_expand_slice(
inputs,
lora_b_weights,
output_tensor,
lora_indices_tensor,
slice_offset,
slice_size,
add_inputs,
)
+76 -25
View File
@@ -62,6 +62,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
captured_lora_counts=captured_lora_counts,
)
# When speculative decoding is enabled, max_num_samples is
# max_batches * (num_speculative_decoding_tokens + 1).
# This line can be optimized by replacing max_num_batched_tokens
# to max_batches * (num_speculative_decoding_tokens + 1).
self.prompt_mapping_meta = LoRAKernelMeta.make(
self.max_loras,
max_num_batched_tokens,
@@ -106,6 +110,14 @@ class PunicaWrapperXPU(PunicaWrapperBase):
add_inputs: bool,
):
token_lora_indices = self._get_token_lora_indices(x)
# After tensor-parallel all-gather (non-fully-sharded LoRA), x may
# have been gathered along the rank dim so x.size(1) == max_lora_rank
# * tp_size, while lora_b only uses max_lora_rank elements. The XPU
# C++ kernel requires inputs.size(1) == lora_b.size(-1), so truncate
# to the actual rank. x[:, :rank] is non-contiguous, hence the copy.
rank = w_t_all.size(-1)
if x.size(1) != rank:
x = x[:, :rank].contiguous()
bgmv_expand_slice(
x, w_t_all, y, token_lora_indices, y_offset, y_slice_size, add_inputs
)
@@ -179,7 +191,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
add_inputs=add_inputs,
)
offset_start += output_slices[slice_idx]
y.view_as(y_org)
y = y.view_as(y_org)
def add_lora_embedding(
self,
@@ -227,7 +239,6 @@ class PunicaWrapperXPU(PunicaWrapperBase):
@ lora_b_stacked[indices[i], layer_idx, :, :]
* scale
).squeeze(0)
Args:
y (torch.Tensor): Output tensor. Will be changed in-place.
x (torch.Tensor): Input tensor
@@ -240,13 +251,17 @@ class PunicaWrapperXPU(PunicaWrapperBase):
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
if buffer is None:
r = lora_b_stacked[0].size(-1)
buffer = torch.zeros( # type: ignore
(len(output_slices), x.size(0), r),
dtype=x.dtype,
device=x.device,
)
assert buffer is None, (
"To minimize overhead, the buffer should be created by "
".add_lora_linear() instead of being passed in."
)
r = lora_b_stacked[0].size(-1)
buffer = torch.zeros( # type: ignore
(len(output_slices), x.size(0), r),
dtype=x.dtype,
device=x.device,
)
add_inputs = kwargs.pop("add_inputs", True)
self.add_shrink(
buffer, # type: ignore
x,
@@ -259,7 +274,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
buffer, # type: ignore
lora_b_stacked,
output_slices,
add_inputs=True,
add_inputs=add_inputs,
**kwargs,
)
@@ -300,12 +315,16 @@ class PunicaWrapperXPU(PunicaWrapperBase):
y = y.view(-1, y.shape[-1])
x = x.view(-1, x.shape[-1])
r = lora_b_stacked.size(-1)
if buffer is None:
buffer = torch.zeros((x.size(0), r), dtype=x.dtype, device=x.device)
assert buffer is None, (
"To minimize overhead, the buffer should be created by "
".add_lora_linear() instead of being passed in."
)
buffer = torch.zeros((x.size(0), r), dtype=x.dtype, device=x.device)
sampler_indices = torch.narrow(self._sampler_indices, 0, 0, x.size(0))
bgmv_shrink(x, lora_a_stacked, buffer, sampler_indices, scale)
bgmv_expand(buffer, lora_b_stacked, y, sampler_indices, add_inputs=True)
return y.view_as(y_org)
y = y.view_as(y_org)
def moe_lora_align_block_size(
self,
@@ -318,33 +337,62 @@ class PunicaWrapperXPU(PunicaWrapperBase):
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
naive_block_assignment: bool = False,
token_lora_mapping: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Aligns tokens and experts into block-sized chunks for LoRA-based
mixture-of-experts (MoE) execution.
When `token_lora_mapping` is provided, it overrides the global mapping
read from `self.token_mapping_meta`. This is how EP+LoRA injects the
per-rank-local token→LoRA map after all-to-all dispatch.
"""
(token_lora_mapping, _, _, _, lora_ids, _, _) = (
self.token_mapping_meta.meta_args(
num_tokens, self.lora_config.specialize_active_lora
)
(
token_lora_mapping_meta,
_,
_,
_,
lora_ids,
_,
_,
) = self.token_mapping_meta.meta_args(
num_tokens, self.lora_config.specialize_active_lora
)
if token_lora_mapping is None:
token_lora_mapping = token_lora_mapping_meta
# Under EP the caller passes local_num_experts but topk_ids carries
# GLOBAL expert indices. The CUDA kernel uses num_experts to size
# its bucketing table; with EP we must size by global_num_experts
# so global topk_ids don't overflow. expert_map inside the kernel
# then translates global→local so the output expert_ids are local
# (mirrors the non-LoRA moe_align_block_size behavior).
kernel_num_experts = (
expert_map.numel() if expert_map is not None else num_experts
)
if naive_block_assignment:
expert_ids = topk_ids.reshape(-1)
sorted_ids = None
num_tokens_post_pad = None
else:
max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
max_num_tokens_padded = topk_ids.numel() + kernel_num_experts * (
block_size - 1
)
if pad_sorted_ids:
max_num_tokens_padded = round_up(max_num_tokens_padded, block_size)
if topk_ids.numel() < kernel_num_experts:
max_num_tokens_padded = topk_ids.numel() * block_size
sorted_ids = torch.empty(
(max_loras * max_num_tokens_padded,),
dtype=torch.int32,
device=topk_ids.device,
)
max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
# Expert ids must be set default to -1 to prevent a blank block
expert_ids = torch.empty(
# Expert ids are initialized to -1 so unused (lora, expert)
# slots don't drive the LoRA Triton kernel into the wrong bucket.
# The kernel overwrites only active slots.
expert_ids = torch.full(
(max_loras * max_num_m_blocks,),
-1,
dtype=torch.int32,
device=topk_ids.device,
)
@@ -355,7 +403,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
ops.moe_lora_align_block_size(
topk_ids,
token_lora_mapping,
num_experts,
kernel_num_experts,
block_size,
max_loras,
max_num_tokens_padded,
@@ -365,11 +413,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
num_tokens_post_pad,
adapter_enabled,
lora_ids,
expert_map,
)
if expert_map is not None:
expert_ids = expert_map[expert_ids]
return None, sorted_ids, expert_ids, num_tokens_post_pad
return token_lora_mapping, sorted_ids, expert_ids, num_tokens_post_pad
def add_lora_fused_moe(
self,
@@ -525,7 +572,8 @@ class PunicaWrapperXPU(PunicaWrapperBase):
SPARSITY_FACTOR = 8
naive_block_assignment = (
expert_map is None
not fully_sharded
and expert_map is None
and num_tokens * top_k * SPARSITY_FACTOR <= local_num_experts * max_loras
)
@@ -543,6 +591,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
adapter_enabled,
expert_map,
naive_block_assignment=naive_block_assignment,
token_lora_mapping=token_lora_mapping,
)
_sorted = sorted_token_ids_lora
@@ -567,6 +616,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
adapter_enabled,
fully_sharded=fully_sharded,
token_lora_mapping=token_lora_mapping,
add_inputs=add_inputs,
)
return (
@@ -680,4 +730,5 @@ class PunicaWrapperXPU(PunicaWrapperBase):
fully_sharded=fully_sharded,
offset=offset,
token_lora_mapping=token_lora_mapping,
add_inputs=add_inputs,
)