diff --git a/tests/lora/test_fused_moe_lora_kernel.py b/tests/lora/test_fused_moe_lora_kernel.py index 8b5b3d577a..11e4d31f87 100644 --- a/tests/lora/test_fused_moe_lora_kernel.py +++ b/tests/lora/test_fused_moe_lora_kernel.py @@ -362,6 +362,83 @@ def test_fused_moe_lora_kernel( torch.testing.assert_close(output, output2, atol=1e-2, rtol=1e-2) +@pytest.mark.parametrize("max_lora_rank", [128, 256]) +@pytest.mark.parametrize("add_inputs", [True, False]) +@pytest.mark.parametrize("num_slices", [1, 2]) +@pytest.mark.parametrize("device", DEVICES) +def test_fused_moe_lora_kernel_large_rank_fallback( + max_lora_rank, + add_inputs, + num_slices, + device, +): + """Ranks above the one-shot limit fall back to shrink/expand kernels.""" + torch.set_default_device(device) + set_random_seed(42) + + num_tokens = 8 + num_sequences = 4 + top_k_num = 2 + num_experts = 4 + max_loras = 2 + N = 256 + K = 256 + block_size = 16 + dtype = torch.bfloat16 + + topk_ids, topk_weights, token_lora_mapping, lora_ids = sample_data( + num_tokens, num_sequences, max_loras, num_experts, top_k_num + ) + lora_a_stacked = [ + torch.rand( + (max_loras, num_experts, max_lora_rank, K), + dtype=dtype, + ) + for _ in range(num_slices) + ] + lora_b_stacked = [ + torch.rand( + (max_loras, num_experts, N // num_slices, max_lora_rank), + dtype=dtype, + ) + for _ in range(num_slices) + ] + hidden_states = torch.rand((num_tokens, K), dtype=dtype) + + residual = torch.randn((num_tokens, top_k_num, N), dtype=dtype) + output = residual.clone() + use_fused_moe_lora_kernel( + topk_ids, + topk_weights, + token_lora_mapping, + max_lora_rank, + top_k_num, + lora_ids, + lora_a_stacked, + lora_b_stacked, + hidden_states, + output, + max_loras, + num_experts, + block_size, + add_inputs=add_inputs, + ) + + expected = use_torch( + hidden_states, + token_lora_mapping, + topk_ids, + lora_a_stacked, + lora_b_stacked, + top_k_num, + num_slices, + ) + if add_inputs: + expected += residual + + torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2) + + def use_fused_moe_lora_kernel_naive( topk_ids, topk_weights, diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py index 703f766889..18bef4406b 100644 --- a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py +++ b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py @@ -106,14 +106,18 @@ def _get_c_ptrs( _LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {} +# The one-shot kernel keeps the rank-dimension intermediate in registers. +# Larger ranks require rank tiling, so route them through the two-kernel path. +_FUSED_MOE_LORA_ONE_SHOT_MAX_RANK = 128 + # --------------------------------------------------------------------------- # Fully-fused MoE-LoRA kernel (one-shot): shrink + expand combined into a single # launch with the rank-dim intermediate kept in registers. Used by the fast -# path of `_fused_moe_lora` for `fully_sharded=False`. The legacy two-kernel -# path (`_fused_moe_lora_kernel` above) is retained for `fully_sharded=True` -# because that path needs to materialise the intermediate cache for an -# all_reduce / all_gather between shrink and expand. +# path of `_fused_moe_lora` for supported ranks with `fully_sharded=False`. +# The legacy two-kernel path (`_fused_moe_lora_kernel` above) is retained for +# larger ranks and for `fully_sharded=True`, where the intermediate cache must +# be materialised for an all_reduce / all_gather between shrink and expand. # --------------------------------------------------------------------------- @@ -349,8 +353,9 @@ def _run_fused_moe_lora_one_shot( # regressed across all M (+8 to +40%): the (64,32) fp32 accumulator + # widened B tile pushed register count past spill threshold, lowering # occupancy by more than the MMA gain saved. - assert rank <= 128, ( - f"fused_moe_lora_one_shot supports max_lora_rank<=128; got rank={rank}" + assert rank <= _FUSED_MOE_LORA_ONE_SHOT_MAX_RANK, ( + "fused_moe_lora_one_shot supports " + f"max_lora_rank<={_FUSED_MOE_LORA_ONE_SHOT_MAX_RANK}; got rank={rank}" ) BLOCK_R = max(triton.next_power_of_2(rank), 16) @@ -1318,6 +1323,7 @@ def _fused_moe_lora_expand( offset: int = 0, use_gdc: bool = False, use_tma: bool = False, + add_inputs: bool = True, ) -> None: b_ptr = _get_ptr(lora_b_stacked, device) K = max_lora_rank @@ -1408,7 +1414,7 @@ def _fused_moe_lora_expand( token_mapping_factor=1, naive_block_assignment=sorted_token_ids is None, MUL_ROUTED_WEIGHT=mul_routed_weight, - ADD_INPUTS=True, + ADD_INPUTS=add_inputs, USE_B_L2_CACHE=True, sort_c=False, IS_PRIMARY=False, @@ -1478,7 +1484,7 @@ def _fused_moe_lora( assert top_k_num == topk_weights.shape[1] # Fast path: single fused kernel - if not fully_sharded: + if not fully_sharded and max_lora_rank <= _FUSED_MOE_LORA_ONE_SHOT_MAX_RANK: M_pairs = topk_weights.numel() if ( sorted_token_ids is None @@ -1524,9 +1530,8 @@ def _fused_moe_lora( ) return - assert add_inputs, ( - "fused_moe_lora(add_inputs=False) is only supported on the " - "fully_sharded=False fast path" + assert add_inputs or not fully_sharded, ( + "fused_moe_lora(add_inputs=False) is not supported with fully_sharded=True" ) device = qcurr_hidden_states.device @@ -1660,6 +1665,7 @@ def _fused_moe_lora( offset, use_gdc=use_gdc, use_tma=use_tma, + add_inputs=add_inputs, )