[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)
Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Dakai An
OpenAI Codex
aoshen02
parent
117cf43fb6
commit
78fdf4efb8
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
@@ -433,6 +434,263 @@ def test_convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
|
||||
assert w13_converted.shape[0] == num_experts
|
||||
assert w2_converted.shape[0] == num_experts
|
||||
assert w13_converted.data_ptr() == w13.data_ptr()
|
||||
assert w2_converted.data_ptr() == w2.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_gated_act_gemm", [True, False])
|
||||
def test_convert_moe_weights_to_flashinfer_trtllm_block_layout_values(
|
||||
is_gated_act_gemm,
|
||||
):
|
||||
from flashinfer.fused_moe.core import (
|
||||
_maybe_get_cached_w3_w1_permute_indices,
|
||||
get_w2_permute_indices_with_cache,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
)
|
||||
|
||||
num_experts, intermediate, hidden = 2, 256, 256
|
||||
w13_multiplier = 2 if is_gated_act_gemm else 1
|
||||
w13 = torch.randn(
|
||||
(num_experts, w13_multiplier * intermediate, hidden),
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
w2 = torch.randn(
|
||||
(num_experts, hidden, intermediate),
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
def _reference_block_layout(
|
||||
weight: torch.Tensor,
|
||||
is_w13: bool,
|
||||
cache: dict[torch.Size, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
outputs = []
|
||||
for expert in weight:
|
||||
expert_uint8 = expert.view(torch.uint8)
|
||||
if is_w13:
|
||||
indices = _maybe_get_cached_w3_w1_permute_indices(
|
||||
cache,
|
||||
expert_uint8,
|
||||
128,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
if is_gated_act_gemm:
|
||||
indices = (indices + expert_uint8.shape[0] // 2) % (
|
||||
expert_uint8.shape[0]
|
||||
)
|
||||
else:
|
||||
indices = get_w2_permute_indices_with_cache(
|
||||
cache,
|
||||
expert_uint8,
|
||||
128,
|
||||
)
|
||||
rows, cols = expert_uint8.shape
|
||||
blocks = expert_uint8.view(rows, cols // 128, 128).permute(1, 0, 2)
|
||||
outputs.append(torch.index_select(blocks, 1, indices.to(weight.device)))
|
||||
return torch.stack(outputs).view(torch.bfloat16)
|
||||
|
||||
reference_cache: dict[torch.Size, torch.Tensor] = {}
|
||||
expected_w13 = _reference_block_layout(w13, is_w13=True, cache=reference_cache)
|
||||
expected_w2 = _reference_block_layout(w2, is_w13=False, cache=reference_cache)
|
||||
w13_ptr = w13.data_ptr()
|
||||
w2_ptr = w2.data_ptr()
|
||||
|
||||
actual_w13, actual_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
w13,
|
||||
w2,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
|
||||
assert actual_w13.data_ptr() == w13_ptr
|
||||
assert actual_w2.data_ptr() == w2_ptr
|
||||
assert torch.equal(actual_w13, expected_w13)
|
||||
assert torch.equal(actual_w2, expected_w2)
|
||||
|
||||
|
||||
def _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, intermediate, is_gated, *, device="cuda"
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=intermediate,
|
||||
activation=MoEActivation.SILU if is_gated else MoEActivation.RELU2_NO_MUL,
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = (intermediate + 127) // 128 * 128
|
||||
method = object.__new__(UnquantizedFusedMoEMethod)
|
||||
method.moe = moe_config
|
||||
method.unquantized_backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
method.moe_kernel = None
|
||||
monkeypatch.setattr(
|
||||
method,
|
||||
"_init_moe_kernel",
|
||||
lambda _: setattr(method, "moe_kernel", object()),
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.moe_config = moe_config
|
||||
with torch.device(device):
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=moe_config.intermediate_size_per_partition,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
return method, layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate", [192, 256])
|
||||
@pytest.mark.parametrize("is_gated", [True, False])
|
||||
def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed(
|
||||
monkeypatch, intermediate, is_gated
|
||||
):
|
||||
"""Each raw reload clears padding left dirty by prior in-place conversion."""
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
)
|
||||
|
||||
method, layer = _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, intermediate, is_gated
|
||||
)
|
||||
w13_shape, w2_shape = layer.w13_weight.shape, layer.w2_weight.shape
|
||||
padded = w2_shape[-1]
|
||||
w13_ptr = layer.w13_weight.data_ptr()
|
||||
w2_ptr = layer.w2_weight.data_ptr()
|
||||
|
||||
for _ in range(2):
|
||||
reloaded_w13 = torch.randn_like(layer.w13_weight)
|
||||
reloaded_w2 = torch.randn_like(layer.w2_weight)
|
||||
reloaded_w13[:, intermediate:padded].zero_()
|
||||
if is_gated:
|
||||
reloaded_w13[:, padded + intermediate :].zero_()
|
||||
reloaded_w2[:, :, intermediate:].zero_()
|
||||
expected_w13, expected_w2 = (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
reloaded_w13.clone(),
|
||||
reloaded_w2.clone(),
|
||||
is_gated_act_gemm=is_gated,
|
||||
)
|
||||
)
|
||||
|
||||
# The loader writes only logical checkpoint slices, not padding.
|
||||
layer.w13_weight.fill_(float("nan"))
|
||||
layer.w2_weight.fill_(float("nan"))
|
||||
layer.w13_weight[:, :intermediate].copy_(reloaded_w13[:, :intermediate])
|
||||
if is_gated:
|
||||
layer.w13_weight[:, padded : padded + intermediate].copy_(
|
||||
reloaded_w13[:, padded : padded + intermediate]
|
||||
)
|
||||
layer.w2_weight[:, :, :intermediate].copy_(reloaded_w2[:, :, :intermediate])
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.w13_weight.shape == w13_shape
|
||||
assert layer.w2_weight.shape == w2_shape
|
||||
assert layer.w13_weight.data_ptr() == w13_ptr
|
||||
assert layer.w2_weight.data_ptr() == w2_ptr
|
||||
kernel_w13, kernel_w2 = method._kernel_weights(layer)
|
||||
assert torch.equal(kernel_w13, expected_w13)
|
||||
assert torch.equal(kernel_w2, expected_w2)
|
||||
|
||||
|
||||
def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_index):
|
||||
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
|
||||
from vllm.model_executor.utils import weights_already_processed
|
||||
|
||||
torch.accelerator.set_device_index(device_index)
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
method, layer = _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, 192, is_gated, device="meta"
|
||||
)
|
||||
loader = object.__new__(IpcModelLoader)
|
||||
loader.mode = mode
|
||||
loader._apply_entries(layer, entries, {}, device_index)
|
||||
pointers = (layer.w13_weight.data_ptr(), layer.w2_weight.data_ptr())
|
||||
|
||||
# Like ipc_cache: a fresh method, only tensor metadata, no _setup_kernel.
|
||||
with weights_already_processed():
|
||||
method.process_weights_after_loading(layer)
|
||||
assert method.moe_kernel is not None
|
||||
actual = method._kernel_weights(layer)
|
||||
for weight, reference, pointer in zip(actual, expected, pointers):
|
||||
assert weight.data_ptr() == pointer
|
||||
assert torch.equal(weight.cpu(), reference)
|
||||
# Cross-process addresses need not match; writes prove sharing/isolation.
|
||||
with torch.no_grad():
|
||||
for weight in actual:
|
||||
weight.fill_(1.0)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_gated", [True, False])
|
||||
@pytest.mark.parametrize("cache_ndim", [3, 4])
|
||||
@pytest.mark.parametrize("mode", ["copy", "zero_copy"])
|
||||
def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state(
|
||||
monkeypatch, is_gated, cache_ndim, mode
|
||||
):
|
||||
"""A spawned IPC consumer restores views without transient method state."""
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import TensorEntry
|
||||
|
||||
method, layer = _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, 192, is_gated, device="meta"
|
||||
)
|
||||
packed = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
torch.randn_like(layer.w13_weight, device="cuda"),
|
||||
torch.randn_like(layer.w2_weight, device="cuda"),
|
||||
is_gated_act_gemm=is_gated,
|
||||
)
|
||||
expected = [weight.cpu() for weight in packed]
|
||||
entries = {}
|
||||
for name, weight in zip(("w13_weight", "w2_weight"), packed):
|
||||
cached = weight.view_as(getattr(layer, name)) if cache_ndim == 3 else weight
|
||||
entries[name] = TensorEntry.from_tensor(cached, kind="param")
|
||||
consumer = multiprocessing.get_context("spawn").Process(
|
||||
target=_check_flashinfer_ipc_weights,
|
||||
args=(
|
||||
entries,
|
||||
expected,
|
||||
is_gated,
|
||||
mode,
|
||||
torch.accelerator.current_device_index(),
|
||||
),
|
||||
)
|
||||
consumer.start()
|
||||
try:
|
||||
# Keep producer allocations alive until the consumer releases its views.
|
||||
consumer.join(timeout=180)
|
||||
assert not consumer.is_alive(), "IPC consumer timed out"
|
||||
assert consumer.exitcode == 0, "IPC consumer failed; see child traceback"
|
||||
for source, reference in zip(packed, expected):
|
||||
expected_source = (
|
||||
torch.ones_like(reference) if mode == "zero_copy" else reference
|
||||
)
|
||||
assert torch.equal(source.cpu(), expected_source)
|
||||
finally:
|
||||
if consumer.is_alive():
|
||||
consumer.terminate()
|
||||
consumer.join(timeout=10)
|
||||
if consumer.is_alive():
|
||||
consumer.kill()
|
||||
consumer.join(timeout=10)
|
||||
consumer.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
Reference in New Issue
Block a user