[LoRA] Initial EP support for LoRA (#40867)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -42,3 +42,10 @@ class MoELoRAContext:
|
||||
# Whether VLLM_TUNED_CONFIG_FOLDER is set; selects get_lora_op_configs vs
|
||||
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
|
||||
use_tuned_config: bool
|
||||
|
||||
# Per-rank token→LoRA mapping after EP dispatch. Set by
|
||||
# FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read
|
||||
# by LoRAExpertsMixin helpers in place of punica_wrapper's global mapping.
|
||||
# None means no dispatch happened (non-EP path), in which case callers
|
||||
# fall back to punica_wrapper.token_mapping_meta.
|
||||
local_token_lora_mapping: torch.Tensor | None = None
|
||||
|
||||
@@ -70,6 +70,7 @@ class LoRAExpertsMixin:
|
||||
lora_context.w13_num_slices,
|
||||
lora_context.fully_sharded,
|
||||
lora_context.use_tuned_config,
|
||||
token_lora_mapping=lora_context.local_token_lora_mapping,
|
||||
)
|
||||
|
||||
def apply_w2_lora(
|
||||
|
||||
@@ -79,9 +79,6 @@ def select_int8_moe_backend(
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
if config.is_lora_enabled:
|
||||
return Int8MoeBackend.TRITON, backend_to_kernel_cls(Int8MoeBackend.TRITON)[0]
|
||||
|
||||
AVAILABLE_BACKENDS = _get_priority_backends(config)
|
||||
|
||||
activation_format = (
|
||||
|
||||
@@ -301,28 +301,6 @@ def select_mxfp4_moe_backend(
|
||||
"""
|
||||
# If activation_key is explicitly provided (e.g., W4A8), use it
|
||||
requested_activation_key = activation_key
|
||||
device_capability = current_platform.get_device_capability()
|
||||
triton_kernels_supported = (
|
||||
has_triton_kernels()
|
||||
and device_capability is not None
|
||||
and (9, 0) <= device_capability < (11, 0)
|
||||
)
|
||||
|
||||
# LoRA: separate experts backend path
|
||||
if config.is_lora_enabled:
|
||||
if not current_platform.is_cuda():
|
||||
# ROCm: Triton mxfp4 LoRA hits GPU memory faults due to
|
||||
# triton_kernels.tensor.Tensor / HIP read-only page issues
|
||||
# during weight swizzle and LoRA forward. Needs work from
|
||||
# the triton_kernels/aiter side.
|
||||
raise NotImplementedError("Mxfp4 LoRA is currently only supported on CUDA.")
|
||||
if envs.VLLM_MXFP4_USE_MARLIN is False and triton_kernels_supported:
|
||||
logger.info_once("Using Triton backend for mxfp4 lora")
|
||||
return Mxfp4MoeBackend.TRITON_UNFUSED, backend_to_kernel_cls(
|
||||
Mxfp4MoeBackend.TRITON_UNFUSED
|
||||
)[0]
|
||||
logger.info_once("Using Marlin backend for mxfp4 lora")
|
||||
return Mxfp4MoeBackend.MARLIN, backend_to_kernel_cls(Mxfp4MoeBackend.MARLIN)[0]
|
||||
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@@ -61,8 +61,6 @@ def select_mxfp8_moe_backend(
|
||||
Returns:
|
||||
A tuple of (fp8_backend, experts_cls).
|
||||
"""
|
||||
if config.is_lora_enabled:
|
||||
raise NotImplementedError("LoRA is not supported for MXFP8 MoE.")
|
||||
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
|
||||
@@ -84,6 +84,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
super().__init__()
|
||||
self.is_sequence_parallel = is_sequence_parallel
|
||||
self._num_dispatchers = num_dispatchers
|
||||
# Set by FusedMoEWithLoRA.set_mapping() when LoRA is active. When
|
||||
# present, prepare() dispatches the per-token LoRA mapping alongside
|
||||
# hidden_states and writes the gathered result back to the context so
|
||||
# experts can use the per-rank-local mapping.
|
||||
self._lora_context = None
|
||||
|
||||
def set_lora_context(self, ctx) -> None:
|
||||
self._lora_context = ctx
|
||||
|
||||
@property
|
||||
def activation_format(self) -> mk.FusedMoEActivationFormat:
|
||||
@@ -124,22 +132,54 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
|
||||
a1q, scales = _quantize_and_setup_dispatch(a1, quant_config, defer_input_quant)
|
||||
|
||||
# When LoRA is active, dispatch the per-token LoRA id along with
|
||||
# hidden_states so every rank receives the correct mapping for the
|
||||
# tokens it ends up processing. The punica_wrapper stores indices as
|
||||
# int64 but the moe_lora_align_block_size kernel expects int32, so
|
||||
# pull the pre-cast view from token_mapping_meta.
|
||||
lora_ctx = self._lora_context
|
||||
local_token_lora_mapping = None
|
||||
if lora_ctx is not None:
|
||||
local_token_lora_mapping = (
|
||||
lora_ctx.punica_wrapper.token_mapping_meta.token_lora_mapping[
|
||||
: a1.shape[0]
|
||||
]
|
||||
)
|
||||
|
||||
extra_tensors: list[torch.Tensor] | None = None
|
||||
if scales is not None:
|
||||
extra_tensors = list(scales)
|
||||
if local_token_lora_mapping is not None:
|
||||
if extra_tensors is None:
|
||||
extra_tensors = []
|
||||
extra_tensors.append(local_token_lora_mapping)
|
||||
|
||||
res = get_ep_group().dispatch(
|
||||
a1q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
extra_tensors=scales,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
|
||||
if scales is None:
|
||||
if extra_tensors is None:
|
||||
assert len(res) == 3
|
||||
a1q, topk_weights, topk_ids = res
|
||||
a1q_scale = None
|
||||
else:
|
||||
assert len(res) == 4
|
||||
a1q, topk_weights, topk_ids, scales = res
|
||||
a1q_scale = _unwrap_scale_and_prepare_for_moe(scales, quant_config)
|
||||
a1q, topk_weights, topk_ids, gathered_extras = res
|
||||
gathered_extras = list(gathered_extras)
|
||||
if local_token_lora_mapping is not None:
|
||||
dispatched_lora_mapping = gathered_extras.pop()
|
||||
assert lora_ctx is not None
|
||||
lora_ctx.local_token_lora_mapping = dispatched_lora_mapping
|
||||
if scales is not None:
|
||||
a1q_scale = _unwrap_scale_and_prepare_for_moe(
|
||||
gathered_extras, quant_config
|
||||
)
|
||||
else:
|
||||
a1q_scale = None
|
||||
|
||||
return a1q, a1q_scale, None, topk_ids, topk_weights
|
||||
|
||||
|
||||
Reference in New Issue
Block a user