[Bugfix] Fix corrupt outputs in MoE FP8 LoRA responses and MoE base model responses when LoRAs are loaded (#42120)
Signed-off-by: Nicholas Edelman <nedelman@nvidia.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Jee Jee Li
Jee Jee Li
parent
560fb8b867
commit
9ea3a4015b
@@ -482,3 +482,127 @@ def test_kernels_hidden_size(
|
||||
seq_length=128,
|
||||
add_inputs=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_add_lora_fused_moe_early_exit(device):
|
||||
"""
|
||||
Ensures add_lora_fused_moe does not invoke the LoRA kernel or
|
||||
modify the output tensor when no_lora_flag_cpu is True
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
|
||||
|
||||
torch.set_default_device(device)
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
max_loras, num_tokens = 4, 16
|
||||
num_experts, top_k, max_lora_rank = 8, 2, 16
|
||||
K, N = 256, 128
|
||||
|
||||
# build PunicaWrapperGPU with minimal lora_config mock
|
||||
lora_config = SimpleNamespace(
|
||||
max_loras=max_loras,
|
||||
specialize_active_lora=False,
|
||||
)
|
||||
wrapper = PunicaWrapperGPU(
|
||||
max_num_batched_tokens=num_tokens,
|
||||
max_batches=num_tokens,
|
||||
device=device,
|
||||
lora_config=lora_config,
|
||||
)
|
||||
|
||||
# simulate a prior LoRA batch so the internal mapping is
|
||||
# populated with stale LoRA IDs
|
||||
lora_mapping = torch.zeros(
|
||||
num_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
lora_mapping[:8] = 1
|
||||
lora_mapping[8:] = 2
|
||||
wrapper.token_mapping_meta.prepare_tensors(lora_mapping)
|
||||
|
||||
# simulate a base-model batch (all -1)
|
||||
base_mapping = torch.full(
|
||||
(num_tokens,),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
wrapper.token_mapping_meta.prepare_tensors(base_mapping)
|
||||
|
||||
assert wrapper.token_mapping_meta.no_lora_flag_cpu[0].item() is True
|
||||
|
||||
# dummy tensors for add_lora_fused_moe
|
||||
y = torch.rand(num_tokens, top_k, N, dtype=torch.bfloat16, device=device)
|
||||
y_snapshot = y.clone()
|
||||
x = torch.rand(num_tokens, K, dtype=torch.bfloat16, device=device)
|
||||
|
||||
lora_a_stacked = (
|
||||
torch.rand(
|
||||
max_loras,
|
||||
num_experts,
|
||||
max_lora_rank,
|
||||
K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
),
|
||||
)
|
||||
lora_b_stacked = (
|
||||
torch.rand(
|
||||
max_loras,
|
||||
num_experts,
|
||||
N,
|
||||
max_lora_rank,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
),
|
||||
)
|
||||
topk_weights = torch.ones(
|
||||
num_tokens,
|
||||
top_k,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
adapter_enabled = torch.ones(
|
||||
max_loras + 1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
shrink_config = expand_config = {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"NUM_WARPS": 4,
|
||||
"NUM_STAGES": 3,
|
||||
"SPLIT_K": 1,
|
||||
}
|
||||
|
||||
# call add_lora_fused_moe - the early exit should prevent any
|
||||
# modification to the output
|
||||
wrapper.add_lora_fused_moe(
|
||||
y=y,
|
||||
x=x,
|
||||
lora_a_stacked=lora_a_stacked,
|
||||
lora_b_stacked=lora_b_stacked,
|
||||
topk_weights=topk_weights,
|
||||
sorted_token_ids=None,
|
||||
expert_ids=torch.zeros(
|
||||
num_tokens * top_k,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
num_tokens_post_padded=None,
|
||||
max_lora_rank=max_lora_rank,
|
||||
top_k_num=top_k,
|
||||
shrink_config=shrink_config,
|
||||
expand_config=expand_config,
|
||||
adapter_enabled=adapter_enabled,
|
||||
)
|
||||
|
||||
assert torch.equal(y, y_snapshot), (
|
||||
"add_lora_fused_moe modified output tensor despite no_lora_flag_cpu=True"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user