[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)
Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Dakai An
OpenAI Codex
aoshen02
parent
117cf43fb6
commit
78fdf4efb8
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
@@ -433,6 +434,263 @@ def test_convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
|
||||
assert w13_converted.shape[0] == num_experts
|
||||
assert w2_converted.shape[0] == num_experts
|
||||
assert w13_converted.data_ptr() == w13.data_ptr()
|
||||
assert w2_converted.data_ptr() == w2.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_gated_act_gemm", [True, False])
|
||||
def test_convert_moe_weights_to_flashinfer_trtllm_block_layout_values(
|
||||
is_gated_act_gemm,
|
||||
):
|
||||
from flashinfer.fused_moe.core import (
|
||||
_maybe_get_cached_w3_w1_permute_indices,
|
||||
get_w2_permute_indices_with_cache,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
)
|
||||
|
||||
num_experts, intermediate, hidden = 2, 256, 256
|
||||
w13_multiplier = 2 if is_gated_act_gemm else 1
|
||||
w13 = torch.randn(
|
||||
(num_experts, w13_multiplier * intermediate, hidden),
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
w2 = torch.randn(
|
||||
(num_experts, hidden, intermediate),
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
def _reference_block_layout(
|
||||
weight: torch.Tensor,
|
||||
is_w13: bool,
|
||||
cache: dict[torch.Size, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
outputs = []
|
||||
for expert in weight:
|
||||
expert_uint8 = expert.view(torch.uint8)
|
||||
if is_w13:
|
||||
indices = _maybe_get_cached_w3_w1_permute_indices(
|
||||
cache,
|
||||
expert_uint8,
|
||||
128,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
if is_gated_act_gemm:
|
||||
indices = (indices + expert_uint8.shape[0] // 2) % (
|
||||
expert_uint8.shape[0]
|
||||
)
|
||||
else:
|
||||
indices = get_w2_permute_indices_with_cache(
|
||||
cache,
|
||||
expert_uint8,
|
||||
128,
|
||||
)
|
||||
rows, cols = expert_uint8.shape
|
||||
blocks = expert_uint8.view(rows, cols // 128, 128).permute(1, 0, 2)
|
||||
outputs.append(torch.index_select(blocks, 1, indices.to(weight.device)))
|
||||
return torch.stack(outputs).view(torch.bfloat16)
|
||||
|
||||
reference_cache: dict[torch.Size, torch.Tensor] = {}
|
||||
expected_w13 = _reference_block_layout(w13, is_w13=True, cache=reference_cache)
|
||||
expected_w2 = _reference_block_layout(w2, is_w13=False, cache=reference_cache)
|
||||
w13_ptr = w13.data_ptr()
|
||||
w2_ptr = w2.data_ptr()
|
||||
|
||||
actual_w13, actual_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
w13,
|
||||
w2,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
|
||||
assert actual_w13.data_ptr() == w13_ptr
|
||||
assert actual_w2.data_ptr() == w2_ptr
|
||||
assert torch.equal(actual_w13, expected_w13)
|
||||
assert torch.equal(actual_w2, expected_w2)
|
||||
|
||||
|
||||
def _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, intermediate, is_gated, *, device="cuda"
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=intermediate,
|
||||
activation=MoEActivation.SILU if is_gated else MoEActivation.RELU2_NO_MUL,
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = (intermediate + 127) // 128 * 128
|
||||
method = object.__new__(UnquantizedFusedMoEMethod)
|
||||
method.moe = moe_config
|
||||
method.unquantized_backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
method.moe_kernel = None
|
||||
monkeypatch.setattr(
|
||||
method,
|
||||
"_init_moe_kernel",
|
||||
lambda _: setattr(method, "moe_kernel", object()),
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.moe_config = moe_config
|
||||
with torch.device(device):
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=moe_config.intermediate_size_per_partition,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
return method, layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate", [192, 256])
|
||||
@pytest.mark.parametrize("is_gated", [True, False])
|
||||
def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed(
|
||||
monkeypatch, intermediate, is_gated
|
||||
):
|
||||
"""Each raw reload clears padding left dirty by prior in-place conversion."""
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
)
|
||||
|
||||
method, layer = _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, intermediate, is_gated
|
||||
)
|
||||
w13_shape, w2_shape = layer.w13_weight.shape, layer.w2_weight.shape
|
||||
padded = w2_shape[-1]
|
||||
w13_ptr = layer.w13_weight.data_ptr()
|
||||
w2_ptr = layer.w2_weight.data_ptr()
|
||||
|
||||
for _ in range(2):
|
||||
reloaded_w13 = torch.randn_like(layer.w13_weight)
|
||||
reloaded_w2 = torch.randn_like(layer.w2_weight)
|
||||
reloaded_w13[:, intermediate:padded].zero_()
|
||||
if is_gated:
|
||||
reloaded_w13[:, padded + intermediate :].zero_()
|
||||
reloaded_w2[:, :, intermediate:].zero_()
|
||||
expected_w13, expected_w2 = (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
reloaded_w13.clone(),
|
||||
reloaded_w2.clone(),
|
||||
is_gated_act_gemm=is_gated,
|
||||
)
|
||||
)
|
||||
|
||||
# The loader writes only logical checkpoint slices, not padding.
|
||||
layer.w13_weight.fill_(float("nan"))
|
||||
layer.w2_weight.fill_(float("nan"))
|
||||
layer.w13_weight[:, :intermediate].copy_(reloaded_w13[:, :intermediate])
|
||||
if is_gated:
|
||||
layer.w13_weight[:, padded : padded + intermediate].copy_(
|
||||
reloaded_w13[:, padded : padded + intermediate]
|
||||
)
|
||||
layer.w2_weight[:, :, :intermediate].copy_(reloaded_w2[:, :, :intermediate])
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.w13_weight.shape == w13_shape
|
||||
assert layer.w2_weight.shape == w2_shape
|
||||
assert layer.w13_weight.data_ptr() == w13_ptr
|
||||
assert layer.w2_weight.data_ptr() == w2_ptr
|
||||
kernel_w13, kernel_w2 = method._kernel_weights(layer)
|
||||
assert torch.equal(kernel_w13, expected_w13)
|
||||
assert torch.equal(kernel_w2, expected_w2)
|
||||
|
||||
|
||||
def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_index):
|
||||
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
|
||||
from vllm.model_executor.utils import weights_already_processed
|
||||
|
||||
torch.accelerator.set_device_index(device_index)
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
method, layer = _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, 192, is_gated, device="meta"
|
||||
)
|
||||
loader = object.__new__(IpcModelLoader)
|
||||
loader.mode = mode
|
||||
loader._apply_entries(layer, entries, {}, device_index)
|
||||
pointers = (layer.w13_weight.data_ptr(), layer.w2_weight.data_ptr())
|
||||
|
||||
# Like ipc_cache: a fresh method, only tensor metadata, no _setup_kernel.
|
||||
with weights_already_processed():
|
||||
method.process_weights_after_loading(layer)
|
||||
assert method.moe_kernel is not None
|
||||
actual = method._kernel_weights(layer)
|
||||
for weight, reference, pointer in zip(actual, expected, pointers):
|
||||
assert weight.data_ptr() == pointer
|
||||
assert torch.equal(weight.cpu(), reference)
|
||||
# Cross-process addresses need not match; writes prove sharing/isolation.
|
||||
with torch.no_grad():
|
||||
for weight in actual:
|
||||
weight.fill_(1.0)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_gated", [True, False])
|
||||
@pytest.mark.parametrize("cache_ndim", [3, 4])
|
||||
@pytest.mark.parametrize("mode", ["copy", "zero_copy"])
|
||||
def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state(
|
||||
monkeypatch, is_gated, cache_ndim, mode
|
||||
):
|
||||
"""A spawned IPC consumer restores views without transient method state."""
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import TensorEntry
|
||||
|
||||
method, layer = _make_unquantized_flashinfer_test_layer(
|
||||
monkeypatch, 192, is_gated, device="meta"
|
||||
)
|
||||
packed = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
torch.randn_like(layer.w13_weight, device="cuda"),
|
||||
torch.randn_like(layer.w2_weight, device="cuda"),
|
||||
is_gated_act_gemm=is_gated,
|
||||
)
|
||||
expected = [weight.cpu() for weight in packed]
|
||||
entries = {}
|
||||
for name, weight in zip(("w13_weight", "w2_weight"), packed):
|
||||
cached = weight.view_as(getattr(layer, name)) if cache_ndim == 3 else weight
|
||||
entries[name] = TensorEntry.from_tensor(cached, kind="param")
|
||||
consumer = multiprocessing.get_context("spawn").Process(
|
||||
target=_check_flashinfer_ipc_weights,
|
||||
args=(
|
||||
entries,
|
||||
expected,
|
||||
is_gated,
|
||||
mode,
|
||||
torch.accelerator.current_device_index(),
|
||||
),
|
||||
)
|
||||
consumer.start()
|
||||
try:
|
||||
# Keep producer allocations alive until the consumer releases its views.
|
||||
consumer.join(timeout=180)
|
||||
assert not consumer.is_alive(), "IPC consumer timed out"
|
||||
assert consumer.exitcode == 0, "IPC consumer failed; see child traceback"
|
||||
for source, reference in zip(packed, expected):
|
||||
expected_source = (
|
||||
torch.ones_like(reference) if mode == "zero_copy" else reference
|
||||
)
|
||||
assert torch.equal(source.cpu(), expected_source)
|
||||
finally:
|
||||
if consumer.is_alive():
|
||||
consumer.terminate()
|
||||
consumer.join(timeout=10)
|
||||
if consumer.is_alive():
|
||||
consumer.kill()
|
||||
consumer.join(timeout=10)
|
||||
consumer.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -12,7 +12,14 @@ correctly handles this mismatch.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
from .utils import make_dummy_moe_config
|
||||
|
||||
@@ -249,6 +256,57 @@ class TestNarrowExpertDataForPadding:
|
||||
class TestWeightLoadingWithPaddedHiddenSize:
|
||||
"""Integration-style tests that simulate padded weight loading."""
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
def test_load_w2_chunks_noncontiguous_cpu_source_to_padded_cuda(
|
||||
self, dtype, tp_rank
|
||||
):
|
||||
hidden = 4096
|
||||
intermediate = 1024
|
||||
loaded_weight = (
|
||||
torch.arange(hidden * intermediate * 2)
|
||||
.remainder(251)
|
||||
.to(dtype)
|
||||
.reshape(hidden, intermediate * 2)
|
||||
)
|
||||
tp_source = loaded_weight[
|
||||
:, intermediate * tp_rank : intermediate * (tp_rank + 1)
|
||||
]
|
||||
expert_data_full = torch.zeros(
|
||||
hidden + 8, intermediate + 8, device="cuda", dtype=dtype
|
||||
)
|
||||
destination = expert_data_full[:hidden, :intermediate]
|
||||
|
||||
assert not tp_source.is_contiguous()
|
||||
assert tp_source.nbytes > 1 << 20
|
||||
assert not destination.is_contiguous()
|
||||
|
||||
experts = object.__new__(RoutedExperts)
|
||||
torch.nn.Module.__init__(experts)
|
||||
experts.moe_config = make_dummy_moe_config()
|
||||
experts.moe_config.moe_parallel_config.tp_size = 2
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
allocated_before = torch.accelerator.memory_allocated()
|
||||
torch.accelerator.reset_peak_memory_stats()
|
||||
experts._load_w2(
|
||||
expert_data=expert_data_full,
|
||||
shard_dim=1,
|
||||
loaded_weight=loaded_weight,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
peak_extra = torch.accelerator.max_memory_allocated() - allocated_before
|
||||
|
||||
# A single copy into the strided CUDA view allocates a full-shard
|
||||
# temporary. Correct values alone would not catch that regression.
|
||||
assert peak_extra < tp_source.nbytes
|
||||
|
||||
torch.testing.assert_close(destination.cpu(), tp_source)
|
||||
assert torch.count_nonzero(expert_data_full[hidden:, :]) == 0
|
||||
assert torch.count_nonzero(expert_data_full[:hidden, intermediate:]) == 0
|
||||
|
||||
def test_load_w2_with_padding(self):
|
||||
"""Simulate loading w2 weights when hidden_size is padded."""
|
||||
padded_hidden = 3072
|
||||
@@ -364,6 +422,74 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
)
|
||||
|
||||
|
||||
class TestUnquantizedTrtLlmPrePadding:
|
||||
@staticmethod
|
||||
def _make_method(
|
||||
intermediate: int,
|
||||
backend: UnquantizedMoeBackend = UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
) -> UnquantizedFusedMoEMethod:
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=2,
|
||||
hidden_dim=64,
|
||||
intermediate_size=intermediate,
|
||||
)
|
||||
method = object.__new__(UnquantizedFusedMoEMethod)
|
||||
method.moe = moe_config
|
||||
method.unquantized_backend = backend
|
||||
method.moe_kernel = None
|
||||
return method
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend,original,expected",
|
||||
[
|
||||
(UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1344, 1408),
|
||||
(UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1408, 1408),
|
||||
(UnquantizedMoeBackend.TRITON, 1344, 1344),
|
||||
],
|
||||
)
|
||||
def test_rounds_intermediate_before_weight_allocation(
|
||||
self,
|
||||
backend: UnquantizedMoeBackend,
|
||||
original: int,
|
||||
expected: int,
|
||||
):
|
||||
method = self._make_method(original, backend)
|
||||
|
||||
hidden, intermediate = method.maybe_roundup_sizes(
|
||||
hidden_size=64,
|
||||
intermediate_size_per_partition=original,
|
||||
act_dtype=torch.bfloat16,
|
||||
moe_parallel_config=method.moe.moe_parallel_config,
|
||||
)
|
||||
|
||||
assert hidden == 64
|
||||
assert intermediate == expected
|
||||
|
||||
@pytest.mark.parametrize("is_act_and_mul", [True, False])
|
||||
def test_create_weights_zeroes_preallocated_padding(self, is_act_and_mul: bool):
|
||||
method = self._make_method(1344)
|
||||
if not is_act_and_mul:
|
||||
method.moe.activation = MoEActivation.RELU2_NO_MUL
|
||||
method.moe.intermediate_size_per_partition = 1408
|
||||
layer = torch.nn.Module()
|
||||
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
num_experts=2,
|
||||
hidden_size=64,
|
||||
intermediate_size_per_partition=1408,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
up_mult = 2 if is_act_and_mul else 1
|
||||
assert layer.w13_weight.shape == (2, up_mult * 1408, 64)
|
||||
assert layer.w2_weight.shape == (2, 64, 1408)
|
||||
assert torch.count_nonzero(layer.w13_weight[:, 1344:1408]) == 0
|
||||
if is_act_and_mul:
|
||||
assert torch.count_nonzero(layer.w13_weight[:, 2752:]) == 0
|
||||
assert torch.count_nonzero(layer.w2_weight[:, :, 1344:]) == 0
|
||||
|
||||
|
||||
class TestLoadWeightsExpertBias:
|
||||
"""Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize
|
||||
all-zero per-expert `.bias` tensors for models whose experts have no bias
|
||||
|
||||
@@ -75,6 +75,10 @@ def test_trtllm_bf16_moe_modular_no_graph(
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
|
||||
# The FlashInfer conversion may rewrite unpadded input storage in place.
|
||||
# Preserve the original layout for the independent torch reference.
|
||||
reference_w1 = w1.clone()
|
||||
reference_w2 = w2.clone()
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
scores = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(scores, topk)
|
||||
@@ -133,8 +137,8 @@ def test_trtllm_bf16_moe_modular_no_graph(
|
||||
|
||||
torch_output = torch_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
reference_w1,
|
||||
reference_w2,
|
||||
score,
|
||||
topk,
|
||||
activation=MoEActivation.SILU,
|
||||
|
||||
@@ -5,6 +5,7 @@ import importlib.machinery
|
||||
import inspect
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from weakref import WeakKeyDictionary, ref
|
||||
|
||||
@@ -467,6 +468,110 @@ def test_model_cleanup(dist_init, default_vllm_config):
|
||||
assert len(mock_info_dict) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_gated", [False, True])
|
||||
@pytest.mark.parametrize("has_bias", [False, True])
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize("padded", [False, True])
|
||||
def test_padded_moe_reload_releases_each_layer(
|
||||
monkeypatch, is_gated, has_bias, tp_rank, padded
|
||||
):
|
||||
"""Checkpoint-sized copies finish each layer without global finalization."""
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
hidden, intermediate, experts = 4, 3, 2
|
||||
stored_hidden, stored_intermediate = (8, 8) if padded else (hidden, intermediate)
|
||||
config = SimpleNamespace(
|
||||
hidden_dim_unpadded=hidden,
|
||||
intermediate_size_per_partition_unpadded=intermediate,
|
||||
is_act_and_mul=is_gated,
|
||||
has_bias=has_bias,
|
||||
tp_rank=tp_rank,
|
||||
moe_parallel_config=SimpleNamespace(tp_size=2),
|
||||
)
|
||||
model = torch.nn.ModuleList()
|
||||
processed: list[torch.nn.Module] = []
|
||||
for _ in range(2):
|
||||
method = object.__new__(UnquantizedFusedMoEMethod)
|
||||
torch.nn.Module.__init__(method)
|
||||
method.moe = config
|
||||
# The regression concerns streaming reload, not kernel conversion.
|
||||
monkeypatch.setattr(method, "process_weights_after_loading", processed.append)
|
||||
layer = object.__new__(RoutedExperts)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.moe_config = config
|
||||
layer.quant_config = None
|
||||
layer.quant_method = method
|
||||
layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i)
|
||||
layer._loaded_expert_biases = set()
|
||||
method.create_weights(
|
||||
layer,
|
||||
experts,
|
||||
stored_hidden,
|
||||
stored_intermediate,
|
||||
torch.float32,
|
||||
weight_loader=layer.weight_loader,
|
||||
)
|
||||
model.append(layer)
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
original_params = [dict(layer.named_parameters()) for layer in model]
|
||||
shards = ["w1", "w3", "w2"] if is_gated else ["w1", "w2"]
|
||||
for cycle in range(2):
|
||||
initialize_layerwise_reload(model)
|
||||
for layer_index, layer in enumerate(model):
|
||||
info = reload_layerwise.get_layerwise_info(layer)
|
||||
inputs = []
|
||||
expected = {
|
||||
name: torch.full_like(p, float("nan"))
|
||||
for name, p in original_params[layer_index].items()
|
||||
}
|
||||
params = dict(layer.named_parameters())
|
||||
calls = [
|
||||
(e, s, b)
|
||||
for e in range(experts)
|
||||
for s in shards
|
||||
for b in ([False, True] if has_bias else [False])
|
||||
]
|
||||
for call_index, (expert, shard, bias) in enumerate(calls):
|
||||
name = ("w2" if shard == "w2" else "w13") + (
|
||||
"_bias" if bias else "_weight"
|
||||
)
|
||||
shape = (
|
||||
((hidden,) if bias else (hidden, 2 * intermediate))
|
||||
if shard == "w2"
|
||||
else ((2 * intermediate,) if bias else (2 * intermediate, hidden))
|
||||
)
|
||||
weight = torch.arange(
|
||||
torch.Size(shape).numel(), dtype=torch.float32
|
||||
).reshape(shape)
|
||||
weight = weight + 100 * (1 + call_index + cycle)
|
||||
inputs.append(ref(weight))
|
||||
# Direct checkpoint loading is the reference for deferred reload.
|
||||
layer.weight_loader(expected[name], weight, name, shard, expert)
|
||||
param = params[name]
|
||||
param.weight_loader(param, weight, name, shard, expert)
|
||||
del weight
|
||||
if call_index != len(calls) - 1:
|
||||
assert info.can_load(), (
|
||||
"Layer processed before its final checkpoint shard"
|
||||
)
|
||||
|
||||
assert not info.can_load(), (
|
||||
"Padding must not defer the layer until finalization"
|
||||
)
|
||||
assert not info.loaded_weights
|
||||
assert len(processed) == cycle * len(model) + layer_index + 1
|
||||
assert all(source() is None for source in inputs)
|
||||
for name, original in original_params[layer_index].items():
|
||||
assert getattr(layer, name) is original
|
||||
# Kernel-specific tests cover padding, which is not checkpoint data.
|
||||
mask = torch.isfinite(expected[name])
|
||||
assert torch.equal(original[mask], expected[name][mask])
|
||||
|
||||
|
||||
def test_get_numel_loaded():
|
||||
param = torch.empty(10, device="meta")
|
||||
loaded_weight = torch.empty(10)
|
||||
|
||||
Reference in New Issue
Block a user