[LoRA] Initial EP support for LoRA (#40867)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-05-09 00:31:23 -07:00
committed by GitHub
parent d6563d693c
commit ecd0b60aad
13 changed files with 209 additions and 73 deletions
@@ -42,3 +42,10 @@ class MoELoRAContext:
# Whether VLLM_TUNED_CONFIG_FOLDER is set; selects get_lora_op_configs vs
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
use_tuned_config: bool
# Per-rank token→LoRA mapping after EP dispatch. Set by
# FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read
# by LoRAExpertsMixin helpers in place of punica_wrapper's global mapping.
# None means no dispatch happened (non-EP path), in which case callers
# fall back to punica_wrapper.token_mapping_meta.
local_token_lora_mapping: torch.Tensor | None = None
@@ -70,6 +70,7 @@ class LoRAExpertsMixin:
lora_context.w13_num_slices,
lora_context.fully_sharded,
lora_context.use_tuned_config,
token_lora_mapping=lora_context.local_token_lora_mapping,
)
def apply_w2_lora(
@@ -79,9 +79,6 @@ def select_int8_moe_backend(
Note: Shape-specific fallbacks may still occur at runtime.
"""
if config.is_lora_enabled:
return Int8MoeBackend.TRITON, backend_to_kernel_cls(Int8MoeBackend.TRITON)[0]
AVAILABLE_BACKENDS = _get_priority_backends(config)
activation_format = (
@@ -301,28 +301,6 @@ def select_mxfp4_moe_backend(
"""
# If activation_key is explicitly provided (e.g., W4A8), use it
requested_activation_key = activation_key
device_capability = current_platform.get_device_capability()
triton_kernels_supported = (
has_triton_kernels()
and device_capability is not None
and (9, 0) <= device_capability < (11, 0)
)
# LoRA: separate experts backend path
if config.is_lora_enabled:
if not current_platform.is_cuda():
# ROCm: Triton mxfp4 LoRA hits GPU memory faults due to
# triton_kernels.tensor.Tensor / HIP read-only page issues
# during weight swizzle and LoRA forward. Needs work from
# the triton_kernels/aiter side.
raise NotImplementedError("Mxfp4 LoRA is currently only supported on CUDA.")
if envs.VLLM_MXFP4_USE_MARLIN is False and triton_kernels_supported:
logger.info_once("Using Triton backend for mxfp4 lora")
return Mxfp4MoeBackend.TRITON_UNFUSED, backend_to_kernel_cls(
Mxfp4MoeBackend.TRITON_UNFUSED
)[0]
logger.info_once("Using Marlin backend for mxfp4 lora")
return Mxfp4MoeBackend.MARLIN, backend_to_kernel_cls(Mxfp4MoeBackend.MARLIN)[0]
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
@@ -61,8 +61,6 @@ def select_mxfp8_moe_backend(
Returns:
A tuple of (fp8_backend, experts_cls).
"""
if config.is_lora_enabled:
raise NotImplementedError("LoRA is not supported for MXFP8 MoE.")
runner_backend = config.moe_backend
if runner_backend != "auto":
@@ -84,6 +84,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
super().__init__()
self.is_sequence_parallel = is_sequence_parallel
self._num_dispatchers = num_dispatchers
# Set by FusedMoEWithLoRA.set_mapping() when LoRA is active. When
# present, prepare() dispatches the per-token LoRA mapping alongside
# hidden_states and writes the gathered result back to the context so
# experts can use the per-rank-local mapping.
self._lora_context = None
def set_lora_context(self, ctx) -> None:
self._lora_context = ctx
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
@@ -124,22 +132,54 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
a1q, scales = _quantize_and_setup_dispatch(a1, quant_config, defer_input_quant)
# When LoRA is active, dispatch the per-token LoRA id along with
# hidden_states so every rank receives the correct mapping for the
# tokens it ends up processing. The punica_wrapper stores indices as
# int64 but the moe_lora_align_block_size kernel expects int32, so
# pull the pre-cast view from token_mapping_meta.
lora_ctx = self._lora_context
local_token_lora_mapping = None
if lora_ctx is not None:
local_token_lora_mapping = (
lora_ctx.punica_wrapper.token_mapping_meta.token_lora_mapping[
: a1.shape[0]
]
)
extra_tensors: list[torch.Tensor] | None = None
if scales is not None:
extra_tensors = list(scales)
if local_token_lora_mapping is not None:
if extra_tensors is None:
extra_tensors = []
extra_tensors.append(local_token_lora_mapping)
res = get_ep_group().dispatch(
a1q,
topk_weights,
topk_ids,
is_sequence_parallel=self.is_sequence_parallel,
extra_tensors=scales,
extra_tensors=extra_tensors,
)
if scales is None:
if extra_tensors is None:
assert len(res) == 3
a1q, topk_weights, topk_ids = res
a1q_scale = None
else:
assert len(res) == 4
a1q, topk_weights, topk_ids, scales = res
a1q_scale = _unwrap_scale_and_prepare_for_moe(scales, quant_config)
a1q, topk_weights, topk_ids, gathered_extras = res
gathered_extras = list(gathered_extras)
if local_token_lora_mapping is not None:
dispatched_lora_mapping = gathered_extras.pop()
assert lora_ctx is not None
lora_ctx.local_token_lora_mapping = dispatched_lora_mapping
if scales is not None:
a1q_scale = _unwrap_scale_and_prepare_for_moe(
gathered_extras, quant_config
)
else:
a1q_scale = None
return a1q, a1q_scale, None, topk_ids, topk_weights