[Bugfix][LoRA] Fall back for high-rank MoE LoRA (#55161)

Signed-off-by: Raina Zhong <yuweiz@nvidia.com>
Co-authored-by: Raina Zhong <yuweiz@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
xiaoyu-xyz
2026-10-02 13:59:28 +03:00
committed by GitHub
co-authored by Raina Zhong mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 98b29aa99a
commit 8c60714da5
2 changed files with 94 additions and 11 deletions
+77
View File
@@ -362,6 +362,83 @@ def test_fused_moe_lora_kernel(
torch.testing.assert_close(output, output2, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("max_lora_rank", [128, 256])
@pytest.mark.parametrize("add_inputs", [True, False])
@pytest.mark.parametrize("num_slices", [1, 2])
@pytest.mark.parametrize("device", DEVICES)
def test_fused_moe_lora_kernel_large_rank_fallback(
max_lora_rank,
add_inputs,
num_slices,
device,
):
"""Ranks above the one-shot limit fall back to shrink/expand kernels."""
torch.set_default_device(device)
set_random_seed(42)
num_tokens = 8
num_sequences = 4
top_k_num = 2
num_experts = 4
max_loras = 2
N = 256
K = 256
block_size = 16
dtype = torch.bfloat16
topk_ids, topk_weights, token_lora_mapping, lora_ids = sample_data(
num_tokens, num_sequences, max_loras, num_experts, top_k_num
)
lora_a_stacked = [
torch.rand(
(max_loras, num_experts, max_lora_rank, K),
dtype=dtype,
)
for _ in range(num_slices)
]
lora_b_stacked = [
torch.rand(
(max_loras, num_experts, N // num_slices, max_lora_rank),
dtype=dtype,
)
for _ in range(num_slices)
]
hidden_states = torch.rand((num_tokens, K), dtype=dtype)
residual = torch.randn((num_tokens, top_k_num, N), dtype=dtype)
output = residual.clone()
use_fused_moe_lora_kernel(
topk_ids,
topk_weights,
token_lora_mapping,
max_lora_rank,
top_k_num,
lora_ids,
lora_a_stacked,
lora_b_stacked,
hidden_states,
output,
max_loras,
num_experts,
block_size,
add_inputs=add_inputs,
)
expected = use_torch(
hidden_states,
token_lora_mapping,
topk_ids,
lora_a_stacked,
lora_b_stacked,
top_k_num,
num_slices,
)
if add_inputs:
expected += residual
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
def use_fused_moe_lora_kernel_naive(
topk_ids,
topk_weights,