[Bugfix][LoRA] Fall back for high-rank MoE LoRA (#55161)
Signed-off-by: Raina Zhong <yuweiz@nvidia.com> Co-authored-by: Raina Zhong <yuweiz@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Raina Zhong
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
98b29aa99a
commit
8c60714da5
@@ -362,6 +362,83 @@ def test_fused_moe_lora_kernel(
|
||||
torch.testing.assert_close(output, output2, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_lora_rank", [128, 256])
|
||||
@pytest.mark.parametrize("add_inputs", [True, False])
|
||||
@pytest.mark.parametrize("num_slices", [1, 2])
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_fused_moe_lora_kernel_large_rank_fallback(
|
||||
max_lora_rank,
|
||||
add_inputs,
|
||||
num_slices,
|
||||
device,
|
||||
):
|
||||
"""Ranks above the one-shot limit fall back to shrink/expand kernels."""
|
||||
torch.set_default_device(device)
|
||||
set_random_seed(42)
|
||||
|
||||
num_tokens = 8
|
||||
num_sequences = 4
|
||||
top_k_num = 2
|
||||
num_experts = 4
|
||||
max_loras = 2
|
||||
N = 256
|
||||
K = 256
|
||||
block_size = 16
|
||||
dtype = torch.bfloat16
|
||||
|
||||
topk_ids, topk_weights, token_lora_mapping, lora_ids = sample_data(
|
||||
num_tokens, num_sequences, max_loras, num_experts, top_k_num
|
||||
)
|
||||
lora_a_stacked = [
|
||||
torch.rand(
|
||||
(max_loras, num_experts, max_lora_rank, K),
|
||||
dtype=dtype,
|
||||
)
|
||||
for _ in range(num_slices)
|
||||
]
|
||||
lora_b_stacked = [
|
||||
torch.rand(
|
||||
(max_loras, num_experts, N // num_slices, max_lora_rank),
|
||||
dtype=dtype,
|
||||
)
|
||||
for _ in range(num_slices)
|
||||
]
|
||||
hidden_states = torch.rand((num_tokens, K), dtype=dtype)
|
||||
|
||||
residual = torch.randn((num_tokens, top_k_num, N), dtype=dtype)
|
||||
output = residual.clone()
|
||||
use_fused_moe_lora_kernel(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
token_lora_mapping,
|
||||
max_lora_rank,
|
||||
top_k_num,
|
||||
lora_ids,
|
||||
lora_a_stacked,
|
||||
lora_b_stacked,
|
||||
hidden_states,
|
||||
output,
|
||||
max_loras,
|
||||
num_experts,
|
||||
block_size,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
|
||||
expected = use_torch(
|
||||
hidden_states,
|
||||
token_lora_mapping,
|
||||
topk_ids,
|
||||
lora_a_stacked,
|
||||
lora_b_stacked,
|
||||
top_k_num,
|
||||
num_slices,
|
||||
)
|
||||
if add_inputs:
|
||||
expected += residual
|
||||
|
||||
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def use_fused_moe_lora_kernel_naive(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
|
||||
@@ -106,14 +106,18 @@ def _get_c_ptrs(
|
||||
|
||||
_LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
|
||||
|
||||
# The one-shot kernel keeps the rank-dimension intermediate in registers.
|
||||
# Larger ranks require rank tiling, so route them through the two-kernel path.
|
||||
_FUSED_MOE_LORA_ONE_SHOT_MAX_RANK = 128
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fully-fused MoE-LoRA kernel (one-shot): shrink + expand combined into a single
|
||||
# launch with the rank-dim intermediate kept in registers. Used by the fast
|
||||
# path of `_fused_moe_lora` for `fully_sharded=False`. The legacy two-kernel
|
||||
# path (`_fused_moe_lora_kernel` above) is retained for `fully_sharded=True`
|
||||
# because that path needs to materialise the intermediate cache for an
|
||||
# all_reduce / all_gather between shrink and expand.
|
||||
# path of `_fused_moe_lora` for supported ranks with `fully_sharded=False`.
|
||||
# The legacy two-kernel path (`_fused_moe_lora_kernel` above) is retained for
|
||||
# larger ranks and for `fully_sharded=True`, where the intermediate cache must
|
||||
# be materialised for an all_reduce / all_gather between shrink and expand.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -349,8 +353,9 @@ def _run_fused_moe_lora_one_shot(
|
||||
# regressed across all M (+8 to +40%): the (64,32) fp32 accumulator +
|
||||
# widened B tile pushed register count past spill threshold, lowering
|
||||
# occupancy by more than the MMA gain saved.
|
||||
assert rank <= 128, (
|
||||
f"fused_moe_lora_one_shot supports max_lora_rank<=128; got rank={rank}"
|
||||
assert rank <= _FUSED_MOE_LORA_ONE_SHOT_MAX_RANK, (
|
||||
"fused_moe_lora_one_shot supports "
|
||||
f"max_lora_rank<={_FUSED_MOE_LORA_ONE_SHOT_MAX_RANK}; got rank={rank}"
|
||||
)
|
||||
BLOCK_R = max(triton.next_power_of_2(rank), 16)
|
||||
|
||||
@@ -1318,6 +1323,7 @@ def _fused_moe_lora_expand(
|
||||
offset: int = 0,
|
||||
use_gdc: bool = False,
|
||||
use_tma: bool = False,
|
||||
add_inputs: bool = True,
|
||||
) -> None:
|
||||
b_ptr = _get_ptr(lora_b_stacked, device)
|
||||
K = max_lora_rank
|
||||
@@ -1408,7 +1414,7 @@ def _fused_moe_lora_expand(
|
||||
token_mapping_factor=1,
|
||||
naive_block_assignment=sorted_token_ids is None,
|
||||
MUL_ROUTED_WEIGHT=mul_routed_weight,
|
||||
ADD_INPUTS=True,
|
||||
ADD_INPUTS=add_inputs,
|
||||
USE_B_L2_CACHE=True,
|
||||
sort_c=False,
|
||||
IS_PRIMARY=False,
|
||||
@@ -1478,7 +1484,7 @@ def _fused_moe_lora(
|
||||
assert top_k_num == topk_weights.shape[1]
|
||||
|
||||
# Fast path: single fused kernel
|
||||
if not fully_sharded:
|
||||
if not fully_sharded and max_lora_rank <= _FUSED_MOE_LORA_ONE_SHOT_MAX_RANK:
|
||||
M_pairs = topk_weights.numel()
|
||||
if (
|
||||
sorted_token_ids is None
|
||||
@@ -1524,9 +1530,8 @@ def _fused_moe_lora(
|
||||
)
|
||||
return
|
||||
|
||||
assert add_inputs, (
|
||||
"fused_moe_lora(add_inputs=False) is only supported on the "
|
||||
"fully_sharded=False fast path"
|
||||
assert add_inputs or not fully_sharded, (
|
||||
"fused_moe_lora(add_inputs=False) is not supported with fully_sharded=True"
|
||||
)
|
||||
|
||||
device = qcurr_hidden_states.device
|
||||
@@ -1660,6 +1665,7 @@ def _fused_moe_lora(
|
||||
offset,
|
||||
use_gdc=use_gdc,
|
||||
use_tma=use_tma,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user