From e33192b26961f765e6923444b3c74250662dde83 Mon Sep 17 00:00:00 2001 From: cwazai <38356712+cwazai@users.noreply.github.com> Date: Mon, 26 Jan 2026 20:56:34 +0800 Subject: [PATCH] =?UTF-8?q?[lora/moe]=20Improve=20fused=20MoE=E2=80=91LoRA?= =?UTF-8?q?=20kernel=20indexing=20and=20memory=20access=20(#32770)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 陈建华 <1647430658@qq.com> Signed-off-by: Yanwen Lin Signed-off-by: kimheesu Signed-off-by: Divakar Verma Signed-off-by: Robert Shaw Signed-off-by: ganyi Signed-off-by: whx-sjtu <2952154980@qq.com> Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Signed-off-by: Nick Hill Signed-off-by: Woosuk Kwon Signed-off-by: Daniel Serebrenik Signed-off-by: Yanan Cao Signed-off-by: Xin Yang Signed-off-by: yewentao256 Signed-off-by: Matthew Wong Signed-off-by: knlnguyen1802 Signed-off-by: Ifta Khairul Alam Adil Signed-off-by: Ifta khairul Alam Adil <25082512+ikaadil@users.noreply.github.com> Signed-off-by: Lucas Wilkinson Signed-off-by: Patrick von Platen Signed-off-by: Huy Do Signed-off-by: Micah Williamson Signed-off-by: Andreas Karatzas Signed-off-by: Kebe Signed-off-by: DarkLight1337 Signed-off-by: Alex Sun Signed-off-by: Isotr0py Signed-off-by: Liran Schour Signed-off-by: liranschour Signed-off-by: wang.yuqi Signed-off-by: NickLucche Signed-off-by: Shengqi Chen Signed-off-by: chaunceyjiang Signed-off-by: Or Ozeri Signed-off-by: Lucas Kabela Signed-off-by: Richard Zou Signed-off-by: Max de Bayser Signed-off-by: Max de Bayser Signed-off-by: AuYang <459461160@qq.com> Signed-off-by: Vadim Gimpelson Signed-off-by: Tyler Michael Smith Signed-off-by: rickychen-infinirc Signed-off-by: Matthew Bonanni Signed-off-by: Fadi Arafeh Signed-off-by: Eldar Kurtic <8884008+eldarkurtic@users.noreply.github.com> Signed-off-by: eldarkurtic <8884008+eldarkurtic@users.noreply.github.com> Signed-off-by: Bill Nell Signed-off-by: RishabhSaini Signed-off-by: Luka Govedič Signed-off-by: Luka Govedič Signed-off-by: Karan Bansal Signed-off-by: jiang1.li Signed-off-by: Li, Jiang Signed-off-by: wang.yuqi Signed-off-by: Tianshu Yu Signed-off-by: raushan Signed-off-by: baonudesifeizhai Signed-off-by: Mark McLoughlin Signed-off-by: sangbumlikeagod Signed-off-by: sangbumlikeagod <98077576+sangbumlikeagod@users.noreply.github.com> Signed-off-by: Matteo Fari Signed-off-by: huanghaoyan.hhy Signed-off-by: Chen Zhang Signed-off-by: Orion Reblitz-Richardson Signed-off-by: Orion Reblitz-Richardson Signed-off-by: marksverdhei Signed-off-by: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Signed-off-by: mgoin Signed-off-by: Randall Smith Signed-off-by: jon Signed-off-by: dolpm <34420038+dolpm@users.noreply.github.com> Signed-off-by: ElizaWszola Signed-off-by: Luka Govedič Signed-off-by: Joe Runde Signed-off-by: mohammad najafi Signed-off-by: Michael Goin Signed-off-by: 7. Sun Signed-off-by: esmeetu Signed-off-by: Roger Wang Signed-off-by: Reagan Signed-off-by: Reagan Lee <96998476+reaganjlee@users.noreply.github.com> Signed-off-by: Hongjian Zhang Signed-off-by: Xingran Wang Signed-off-by: Hiroken. <105287758+HirokenOvo@users.noreply.github.com> Signed-off-by: Lukas Geiger Signed-off-by: Tsai, Louie Signed-off-by: Louie Tsai Signed-off-by: Maryam Tahhan Signed-off-by: Joshua Deng Signed-off-by: tjtanaa Signed-off-by: LopezCastroRoberto Signed-off-by: JJJYmmm <92386084+JJJYmmm@users.noreply.github.com> Signed-off-by: Jee Jee Li Signed-off-by: cwazai <38356712+cwazai@users.noreply.github.com> Co-authored-by: Yanwen Lin Co-authored-by: Kim Hee Su Co-authored-by: Divakar Verma <137818590+divakar-amd@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw Co-authored-by: Pleaplusone Co-authored-by: whx <56632993+whx-sjtu@users.noreply.github.com> Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Nick Hill Co-authored-by: Woosuk Kwon Co-authored-by: danisereb Co-authored-by: Yanan Cao Co-authored-by: Xin Yang <105740670+xyang16@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Matt <156021403+mawong-amd@users.noreply.github.com> Co-authored-by: knlnguyen1802 Co-authored-by: Lucain Co-authored-by: Ifta khairul Alam Adil <25082512+ikaadil@users.noreply.github.com> Co-authored-by: Lucas Wilkinson Co-authored-by: Patrick von Platen Co-authored-by: Huy Do Co-authored-by: Micah Williamson Co-authored-by: Andreas Karatzas Co-authored-by: Matthew Wong Co-authored-by: Kebe Co-authored-by: Cyrus Leung Co-authored-by: Alex Sun Co-authored-by: Isotr0py Co-authored-by: liranschour Co-authored-by: Or Ozeri Co-authored-by: wang.yuqi Co-authored-by: Nicolò Lucchesi Co-authored-by: Shengqi Chen Co-authored-by: Chauncey Co-authored-by: Or Ozeri Co-authored-by: Lucas Kabela Co-authored-by: Richard Zou Co-authored-by: Maximilien de Bayser Co-authored-by: Xu Jinyang <72930776+AuYang261@users.noreply.github.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Tyler Michael Smith Co-authored-by: David Ramon Prados Co-authored-by: RickyChen / 陳昭儒 Co-authored-by: Matthew Bonanni Co-authored-by: Fadi Arafeh <115173828+fadara01@users.noreply.github.com> Co-authored-by: Eldar Kurtić <8884008+eldarkurtic@users.noreply.github.com> Co-authored-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Rishabh Saini Co-authored-by: Luka Govedič Co-authored-by: Michael Goin Co-authored-by: Karan Bansal Co-authored-by: Li, Jiang Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: tianshu-Michael-yu <101950379+tianshu-Michael-yu@users.noreply.github.com> Co-authored-by: Raushan Turganbay Co-authored-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Co-authored-by: Mark McLoughlin Co-authored-by: sangbumlikeagod <98077576+sangbumlikeagod@users.noreply.github.com> Co-authored-by: Matteo Fari Co-authored-by: Harry Huang Co-authored-by: Chen Zhang Co-authored-by: Orion Reblitz-Richardson Co-authored-by: Kevin H. Luu Co-authored-by: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Co-authored-by: Claude Opus 4.5 Co-authored-by: rasmith Co-authored-by: Randall Smith Co-authored-by: joninco Co-authored-by: dolpm <34420038+dolpm@users.noreply.github.com> Co-authored-by: ElizaWszola Co-authored-by: Varun Sundar Rabindranath Co-authored-by: Luka Govedič Co-authored-by: Lucas Wilkinson Co-authored-by: Luka Govedič Co-authored-by: Joe Runde Co-authored-by: monajafi-amd Co-authored-by: ruizcrp Co-authored-by: Shengqi Chen Co-authored-by: 7. Sun Co-authored-by: Roy Wang Co-authored-by: Roger Wang Co-authored-by: Reagan Lee <96998476+reaganjlee@users.noreply.github.com> Co-authored-by: Hiroken. <105287758+HirokenOvo@users.noreply.github.com> Co-authored-by: Xingran Wang Co-authored-by: david guan <102001211+Chenhao-Guan@users.noreply.github.com> Co-authored-by: Lukas Geiger Co-authored-by: Louie Tsai Co-authored-by: Maryam Tahhan Co-authored-by: Joshua Deng <91448271+joshuadeng@users.noreply.github.com> Co-authored-by: Nick Hill Co-authored-by: TJian Co-authored-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: JJJYmmm <92386084+JJJYmmm@users.noreply.github.com> Co-authored-by: Jee Jee Li --- vllm/lora/ops/triton_ops/fused_moe_lora_op.py | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py index d3bcacf74f..ff06725cd6 100644 --- a/vllm/lora/ops/triton_ops/fused_moe_lora_op.py +++ b/vllm/lora/ops/triton_ops/fused_moe_lora_op.py @@ -62,6 +62,7 @@ def _fused_moe_lora_kernel( num_experts, lora_ids, adapter_enabled, + max_loras, # <<< PR2: rename, used for masks when grid axis-2 != max_loras # The stride variables represent how much to increase the ptr by when # moving by 1 element in a particular dimension. E.g. `stride_am` is # how much to increase `a_ptr` by to get the element one row down @@ -83,6 +84,7 @@ def _fused_moe_lora_kernel( num_slice_c: tl.constexpr, top_k: tl.constexpr, MUL_ROUTED_WEIGHT: tl.constexpr, + USE_B_L2_CACHE: tl.constexpr, # new, enable .ca load for B BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, @@ -104,10 +106,13 @@ def _fused_moe_lora_kernel( if moe_enabled == 0: # Early exit for the no moe lora case. return - # The grid size on axis 2 is (max_loras + 1) to handle the no-lora case - # (lora_id == -1), but sorted_token_ids and expert_ids are allocated with - # shape (max_loras, ...). Use (num_programs - 1) for correct bounds checking. - max_loras = tl.num_programs(axis=2) - 1 + # The grid's axis-2 dimension is max_loras + 1 to accommodate the -1 sentinel. + # This guard ensures we don't access sorted_token_ids / expert_ids / + # num_tokens_post_padded beyond their allocated bounds if an invalid + # lora_id somehow appears. Although the caller should pass correct + # max_loras, defensive programming prevents accidental out-of-bounds. + if lora_id >= max_loras: + return grid_k = tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K) # calculate pid_m,pid_n @@ -136,10 +141,11 @@ def _fused_moe_lora_kernel( cur_b_ptr = tl.load(b_ptr + slice_id).to(tl.pointer_type(c_ptr.dtype.element_ty)) cur_c_ptr = c_ptr + (slice_id % num_slice_c) * slice_c_size - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N + # remove modulo wrap-around + offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int32) offs_k = pid_sk * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) - offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int32) token_ind = stride_tl * lora_id + offs_token_id offs_token = tl.load( sorted_token_ids_ptr + token_ind, @@ -176,7 +182,13 @@ def _fused_moe_lora_kernel( # GDC wait waits for ALL programs in the prior kernel to complete # before continuing. # pre-fetch lora weight - b = tl.load(b_ptrs, mask=offs_k[:, None] < k_remaining, other=0.0) + # add (offs_bn < N) mask; optional .ca for B + b_mask = (offs_k[:, None] < k_remaining) & (offs_bn[None, :] < N) + if USE_B_L2_CACHE: + b = tl.load(b_ptrs, mask=b_mask, other=0.0, cache_modifier=".ca") + else: + b = tl.load(b_ptrs, mask=b_mask, other=0.0) + if USE_GDC and not IS_PRIMARY: tl.extra.cuda.gdc_wait() a = tl.load( @@ -276,6 +288,7 @@ def _fused_moe_lora_shrink( num_experts, lora_ids, adapter_enabled, + lora_a_stacked[0].shape[0], qcurr_hidden_states.stride(0), qcurr_hidden_states.stride(1), w1_lora_a_stacked.stride(0), @@ -292,6 +305,7 @@ def _fused_moe_lora_shrink( num_slice_c=num_slices, top_k=1 if mul_routed_weight else top_k_num, MUL_ROUTED_WEIGHT=False, + USE_B_L2_CACHE=True, # new IS_PRIMARY=True, **shrink_config, ) @@ -377,6 +391,7 @@ def _fused_moe_lora_expand( num_experts, lora_ids, adapter_enabled, + lora_b_stacked[0].shape[0], a_intermediate_cache1.stride(0), a_intermediate_cache1.stride(1), w1_lora_b_stacked.stride(0), @@ -393,6 +408,7 @@ def _fused_moe_lora_expand( num_slice_c=num_slices, top_k=1, MUL_ROUTED_WEIGHT=mul_routed_weight, + USE_B_L2_CACHE=True, # new IS_PRIMARY=False, **expand_config, )