[Bugfix] Fix corrupt outputs in MoE FP8 LoRA responses and MoE base model responses when LoRAs are loaded (#42120)

Signed-off-by: Nicholas Edelman <nedelman@nvidia.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
nv-nedelman-1
2026-06-18 18:26:09 -07:00
committed by GitHub
co-authored by Jee Jee Li Jee Jee Li
parent 560fb8b867
commit 9ea3a4015b
5 changed files with 196 additions and 4 deletions
@@ -59,3 +59,10 @@ class MoELoRAContext:
# None means no dispatch happened (non-EP path), in which case callers
# fall back to punica_wrapper.token_mapping_meta.
local_token_lora_mapping: torch.Tensor | None = None
# Original unquantized hidden states, stashed by the modular kernel
# before the prepare step potentially quantizes them. Used by
# apply_w13_lora so the LoRA kernel sees correct-magnitude activations
# instead of raw quantized values that are missing the activation scale.
# Set per forward pass; None until the modular kernel writes it.
original_hidden_states: torch.Tensor | None = None
@@ -77,6 +77,16 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@property
def expects_unquantized_inputs(self) -> bool:
# Defer activation quantization to apply() only when LoRA is active AND
# tokens are dispatched across ranks (DP+EP all2all).
return (
self._lora_context is not None
and self.quant_dtype is not None
and self.moe_config.moe_parallel_config.use_all2all_kernels
)
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cuda_alike() or current_platform.is_xpu()
@@ -223,6 +233,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
torch.float8_e4m3fnuz,
]
# We declared expects_unquantized_inputs (LoRA + DP/EP all2all), so the
# prepare step deferred activation quantization to this kernel:
# `hidden_states` arrives unquantized. Keep the unquantized tensor for
# the LoRA shrink input and quantize a copy here for the base GEMM
# (mirrors what the prepare step would have done, but after the
# all-gather so the layout matches the gathered topk_ids / token map).
lora_unquantized_hidden_states: torch.Tensor | None = None
if self.expects_unquantized_inputs:
assert a1q_scale is None
lora_unquantized_hidden_states = hidden_states
hidden_states, a1q_scale = moe_kernel_quantize_input(
hidden_states,
self.a1_scale,
self.quant_dtype,
self.per_act_token_quant,
self.block_shape,
quantization_emulation=self.quantization_emulation,
)
E, num_tokens, N, K, top_k_num = self.moe_problem_size(
hidden_states, w1, w2, topk_ids
)
@@ -280,12 +309,28 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
# GEMM on the default stream and the LoRA fast-path on aux_stream;
# the LoRA writes its delta into a fresh zero buffer (add_inputs=
# False) and we sum it into intermediate_cache1 after both finish.
#
# The LoRA shrink kernel needs unquantized, gathered-layout
# activations. When activation quant was deferred to this kernel
# (expects_unquantized_inputs), the input we quantized above is exactly
# that, so use it directly. Otherwise fall back to the context stash
# (e.g. weight-only quant), guarding on a row-count match so a
# DP-gathered layout never indexes a local stash out of bounds.
sorted_token_ids_lora = None
expert_ids_lora = None
num_tokens_post_padded_lora = None
token_lora_mapping = None
lora_context = self._lora_context
if lora_unquantized_hidden_states is not None:
lora_x = lora_unquantized_hidden_states
elif (
lora_context is not None
and lora_context.original_hidden_states is not None
and lora_context.original_hidden_states.shape[0] == hidden_states.shape[0]
):
lora_x = lora_context.original_hidden_states
else:
lora_x = hidden_states
def _base_w13_fn():
invoke_fused_moe_triton_kernel(
@@ -322,7 +367,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
return self.apply_w13_lora(
lora_context,
y=lora_delta_w13,
x=hidden_states,
x=lora_x,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
@@ -359,7 +404,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
) = self.apply_w13_lora(
lora_context,
y=intermediate_cache1,
x=hidden_states,
x=lora_x,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
@@ -1407,6 +1407,13 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input,
)
# Stash the original unquantized hidden states on the LoRA context
# so apply_w13_lora sees correct-magnitude activations instead of
# the potentially quantized values produced by _prepare().
lora_ctx = getattr(self.fused_experts, "_lora_context", None)
if lora_ctx is not None:
lora_ctx.original_hidden_states = hidden_states
fused_out = self._fused_experts(
in_dtype=hidden_states.dtype,
a1q=a1q,
@@ -1424,6 +1431,9 @@ class FusedMoEKernelModularImpl:
output_alias=output,
)
if lora_ctx is not None:
lora_ctx.original_hidden_states = None
return self._finalize(
output,
fused_out,