[Bugfix] Fix corrupt outputs in MoE FP8 LoRA responses and MoE base model responses when LoRAs are loaded (#42120)

Signed-off-by: Nicholas Edelman <nedelman@nvidia.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
nv-nedelman-1
2026-06-18 18:26:09 -07:00
committed by GitHub
co-authored by Jee Jee Li Jee Jee Li
parent 560fb8b867
commit 9ea3a4015b
5 changed files with 196 additions and 4 deletions
+124
View File
@@ -482,3 +482,127 @@ def test_kernels_hidden_size(
seq_length=128,
add_inputs=True,
)
@pytest.mark.parametrize("device", DEVICES)
def test_add_lora_fused_moe_early_exit(device):
"""
Ensures add_lora_fused_moe does not invoke the LoRA kernel or
modify the output tensor when no_lora_flag_cpu is True
"""
from types import SimpleNamespace
from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
torch.set_default_device(device)
torch.accelerator.set_device_index(device)
max_loras, num_tokens = 4, 16
num_experts, top_k, max_lora_rank = 8, 2, 16
K, N = 256, 128
# build PunicaWrapperGPU with minimal lora_config mock
lora_config = SimpleNamespace(
max_loras=max_loras,
specialize_active_lora=False,
)
wrapper = PunicaWrapperGPU(
max_num_batched_tokens=num_tokens,
max_batches=num_tokens,
device=device,
lora_config=lora_config,
)
# simulate a prior LoRA batch so the internal mapping is
# populated with stale LoRA IDs
lora_mapping = torch.zeros(
num_tokens,
dtype=torch.int32,
device=device,
)
lora_mapping[:8] = 1
lora_mapping[8:] = 2
wrapper.token_mapping_meta.prepare_tensors(lora_mapping)
# simulate a base-model batch (all -1)
base_mapping = torch.full(
(num_tokens,),
-1,
dtype=torch.int32,
device=device,
)
wrapper.token_mapping_meta.prepare_tensors(base_mapping)
assert wrapper.token_mapping_meta.no_lora_flag_cpu[0].item() is True
# dummy tensors for add_lora_fused_moe
y = torch.rand(num_tokens, top_k, N, dtype=torch.bfloat16, device=device)
y_snapshot = y.clone()
x = torch.rand(num_tokens, K, dtype=torch.bfloat16, device=device)
lora_a_stacked = (
torch.rand(
max_loras,
num_experts,
max_lora_rank,
K,
dtype=torch.bfloat16,
device=device,
),
)
lora_b_stacked = (
torch.rand(
max_loras,
num_experts,
N,
max_lora_rank,
dtype=torch.bfloat16,
device=device,
),
)
topk_weights = torch.ones(
num_tokens,
top_k,
dtype=torch.float32,
device=device,
)
adapter_enabled = torch.ones(
max_loras + 1,
dtype=torch.int32,
device=device,
)
shrink_config = expand_config = {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"NUM_WARPS": 4,
"NUM_STAGES": 3,
"SPLIT_K": 1,
}
# call add_lora_fused_moe - the early exit should prevent any
# modification to the output
wrapper.add_lora_fused_moe(
y=y,
x=x,
lora_a_stacked=lora_a_stacked,
lora_b_stacked=lora_b_stacked,
topk_weights=topk_weights,
sorted_token_ids=None,
expert_ids=torch.zeros(
num_tokens * top_k,
dtype=torch.int32,
device=device,
),
num_tokens_post_padded=None,
max_lora_rank=max_lora_rank,
top_k_num=top_k,
shrink_config=shrink_config,
expand_config=expand_config,
adapter_enabled=adapter_enabled,
)
assert torch.equal(y, y_snapshot), (
"add_lora_fused_moe modified output tensor despite no_lora_flag_cpu=True"
)