[MoE] Encapsulate TRT-LLM BF16 weight layout handling (#57405)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -12,6 +12,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -27,6 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
|
||||
TrtLlmBf16LoRAExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
convert_to_unquantized_kernel_format,
|
||||
@@ -150,3 +154,48 @@ def test_trtllm_bf16_moe_modular_no_graph(
|
||||
atol=1e-1,
|
||||
rtol=2e-1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_lora_delta", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_bf16_lora_accepts_checkpoint_shaped_weights(has_lora_delta):
|
||||
"""LoRA dispatch accepts both 3D parameters and legacy 4D packed views."""
|
||||
config = make_dummy_moe_config(
|
||||
num_experts=128, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
experts = TrtLlmBf16LoRAExperts(config, FUSED_MOE_UNQUANTIZED_CONFIG)
|
||||
x = torch.randn(2, 256, device="cuda", dtype=torch.bfloat16) / 10
|
||||
w1, w2 = convert_to_unquantized_kernel_format(
|
||||
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
config,
|
||||
torch.randn(128, 256, 256, device="cuda", dtype=torch.bfloat16) / 10,
|
||||
torch.randn(128, 256, 128, device="cuda", dtype=torch.bfloat16) / 10,
|
||||
)
|
||||
assert w1.ndim == w2.ndim == 3
|
||||
topk = (
|
||||
torch.tensor([[0], [1]], device="cuda", dtype=torch.int32),
|
||||
torch.ones(2, 1, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
delta = torch.randn(2, 1, 256, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def invoke(w1, w2):
|
||||
result = experts.invoke_routed_moe(
|
||||
hidden_states=x,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_ids_and_weights=topk,
|
||||
gemm1_lora_delta=delta if has_lora_delta else None,
|
||||
global_num_experts=128,
|
||||
a1q_scale=None,
|
||||
output=torch.empty_like(x),
|
||||
)
|
||||
if has_lora_delta:
|
||||
# Only compare rows belonging to real tokens, excluding padding.
|
||||
indices = result[2].flatten().long()
|
||||
return result[0][indices], result[3][indices]
|
||||
return result
|
||||
|
||||
actual = invoke(w1, w2)
|
||||
expected = invoke(w1.view(128, 4, 256, 64), w2.view(128, 2, 256, 64))
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
|
||||
Reference in New Issue
Block a user