[MoE] Encapsulate TRT-LLM BF16 weight layout handling (#57405)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-09-17 17:47:30 -04:00
committed by GitHub
parent db7a24c230
commit ac78445bc9
7 changed files with 138 additions and 117 deletions
+49
View File
@@ -12,6 +12,7 @@ import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.utils import torch_moe
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
@@ -27,6 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsModular,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
TrtLlmBf16LoRAExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
convert_to_unquantized_kernel_format,
@@ -150,3 +154,48 @@ def test_trtllm_bf16_moe_modular_no_graph(
atol=1e-1,
rtol=2e-1,
)
@pytest.mark.parametrize("has_lora_delta", [False, True])
@torch.inference_mode()
def test_trtllm_bf16_lora_accepts_checkpoint_shaped_weights(has_lora_delta):
"""LoRA dispatch accepts both 3D parameters and legacy 4D packed views."""
config = make_dummy_moe_config(
num_experts=128, hidden_dim=256, intermediate_size=128
)
experts = TrtLlmBf16LoRAExperts(config, FUSED_MOE_UNQUANTIZED_CONFIG)
x = torch.randn(2, 256, device="cuda", dtype=torch.bfloat16) / 10
w1, w2 = convert_to_unquantized_kernel_format(
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
config,
torch.randn(128, 256, 256, device="cuda", dtype=torch.bfloat16) / 10,
torch.randn(128, 256, 128, device="cuda", dtype=torch.bfloat16) / 10,
)
assert w1.ndim == w2.ndim == 3
topk = (
torch.tensor([[0], [1]], device="cuda", dtype=torch.int32),
torch.ones(2, 1, device="cuda", dtype=torch.float32),
)
delta = torch.randn(2, 1, 256, device="cuda", dtype=torch.bfloat16)
def invoke(w1, w2):
result = experts.invoke_routed_moe(
hidden_states=x,
w1=w1,
w2=w2,
topk_ids_and_weights=topk,
gemm1_lora_delta=delta if has_lora_delta else None,
global_num_experts=128,
a1q_scale=None,
output=torch.empty_like(x),
)
if has_lora_delta:
# Only compare rows belonging to real tokens, excluding padding.
indices = result[2].flatten().long()
return result[0][indices], result[3][indices]
return result
actual = invoke(w1, w2)
expected = invoke(w1.view(128, 4, 256, 64), w2.view(128, 2, 256, 64))
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)