[MoE] Encapsulate TRT-LLM BF16 weight layout handling (#57405)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-09-17 17:47:30 -04:00
committed by GitHub
parent db7a24c230
commit ac78445bc9
7 changed files with 138 additions and 117 deletions
+21 -9
View File
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
view_as_block_major_k,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsModular,
TrtLlmFp8ExpertsMonolithic,
@@ -570,6 +573,10 @@ def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed(
padded = w2_shape[-1]
w13_ptr = layer.w13_weight.data_ptr()
w2_ptr = layer.w2_weight.data_ptr()
load_numel = (
layer.w13_weight.weight_loader_numel,
layer.w2_weight.weight_loader_numel,
)
for _ in range(2):
reloaded_w13 = torch.randn_like(layer.w13_weight)
@@ -602,19 +609,24 @@ def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed(
assert layer.w2_weight.shape == w2_shape
assert layer.w13_weight.data_ptr() == w13_ptr
assert layer.w2_weight.data_ptr() == w2_ptr
kernel_w13, kernel_w2 = method._kernel_weights(layer)
assert (
layer.w13_weight.weight_loader_numel,
layer.w2_weight.weight_loader_numel,
) == load_numel
kernel_w13 = view_as_block_major_k(layer.w13_weight)
kernel_w2 = view_as_block_major_k(layer.w2_weight)
assert torch.equal(kernel_w13, expected_w13)
assert torch.equal(kernel_w2, expected_w2)
def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_index):
def _check_flashinfer_ipc_weights(entries, expected, mode, device_index):
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
from vllm.model_executor.utils import weights_already_processed
torch.accelerator.set_device_index(device_index)
with pytest.MonkeyPatch.context() as monkeypatch:
method, layer = _make_unquantized_flashinfer_test_layer(
monkeypatch, 192, is_gated, device="meta"
monkeypatch, 192, is_gated=True, device="meta"
)
loader = object.__new__(IpcModelLoader)
loader.mode = mode
@@ -625,7 +637,10 @@ def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_inde
with weights_already_processed():
method.process_weights_after_loading(layer)
assert method.moe_kernel is not None
actual = method._kernel_weights(layer)
actual = (
view_as_block_major_k(layer.w13_weight),
view_as_block_major_k(layer.w2_weight),
)
for weight, reference, pointer in zip(actual, expected, pointers):
assert weight.data_ptr() == pointer
assert torch.equal(weight.cpu(), reference)
@@ -636,11 +651,10 @@ def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_inde
torch.accelerator.synchronize()
@pytest.mark.parametrize("is_gated", [True, False])
@pytest.mark.parametrize("cache_ndim", [3, 4])
@pytest.mark.parametrize("mode", ["copy", "zero_copy"])
def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state(
monkeypatch, is_gated, cache_ndim, mode
monkeypatch, cache_ndim, mode
):
"""A spawned IPC consumer restores views without transient method state."""
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
@@ -649,13 +663,12 @@ def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state(
from vllm.model_executor.model_loader.weight_cache.protocol import TensorEntry
method, layer = _make_unquantized_flashinfer_test_layer(
monkeypatch, 192, is_gated, device="meta"
monkeypatch, 192, is_gated=True, device="meta"
)
packed = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
torch.randn_like(layer.w13_weight, device="cuda"),
torch.randn_like(layer.w2_weight, device="cuda"),
is_gated_act_gemm=is_gated,
)
expected = [weight.cpu() for weight in packed]
entries = {}
@@ -667,7 +680,6 @@ def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state(
args=(
entries,
expected,
is_gated,
mode,
torch.accelerator.current_device_index(),
),