Add routed expert loading for gpt-oss (#52209)
Signed-off-by: Wyett <wyettzeng@gmail.com> Signed-off-by: wyett <wyettzeng@gmail.com>
This commit is contained in:
@@ -0,0 +1,152 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.model_executor.models.gpt_oss import (
|
||||||
|
GptOssForCausalLM,
|
||||||
|
GptOssModel,
|
||||||
|
GptOssRoutedExperts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _TestExperts(GptOssRoutedExperts):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
param_name: str,
|
||||||
|
shape: tuple[int, ...],
|
||||||
|
*,
|
||||||
|
tp_rank: int = 0,
|
||||||
|
quant_method_name: str = "TestQuantMethod",
|
||||||
|
weight_dtype: str = "",
|
||||||
|
dtype: torch.dtype = torch.float32,
|
||||||
|
) -> None:
|
||||||
|
torch.nn.Module.__init__(self)
|
||||||
|
self.moe_config = SimpleNamespace(
|
||||||
|
moe_parallel_config=SimpleNamespace(tp_size=2, tp_rank=tp_rank)
|
||||||
|
)
|
||||||
|
self.quant_method = type(
|
||||||
|
quant_method_name, (), {"weight_dtype": weight_dtype}
|
||||||
|
)()
|
||||||
|
|
||||||
|
param = torch.nn.Parameter(torch.zeros(shape, dtype=dtype), requires_grad=False)
|
||||||
|
param.weight_loader = self.weight_loader
|
||||||
|
self.register_parameter(param_name, param)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _map_global_expert_id_to_local_expert_id(expert_id: int) -> int:
|
||||||
|
return {3: 0, 7: 1}.get(expert_id, -1)
|
||||||
|
|
||||||
|
|
||||||
|
def _load(
|
||||||
|
experts: _TestExperts,
|
||||||
|
param_name: str,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
shard_id: str,
|
||||||
|
expert_id: int = 3,
|
||||||
|
) -> bool:
|
||||||
|
param = getattr(experts, param_name)
|
||||||
|
return param.weight_loader(
|
||||||
|
param,
|
||||||
|
loaded_weight,
|
||||||
|
weight_name=f"model.layers.0.mlp.experts.routed_experts.{param_name}",
|
||||||
|
shard_id=shard_id,
|
||||||
|
expert_id=expert_id,
|
||||||
|
return_success=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("base_layer_prefix", ["", "base_layer."])
|
||||||
|
def test_streamed_expert_resolves_wrapped_parameter(base_layer_prefix: str) -> None:
|
||||||
|
experts = _TestExperts("w13_weight", (2, 4, 3))
|
||||||
|
checkpoint_name = "model.layers.2.mlp.experts.7.gate_up_proj"
|
||||||
|
mapped_name = GptOssForCausalLM.hf_to_vllm_mapper.apply_list([checkpoint_name])[0]
|
||||||
|
fused_name = (
|
||||||
|
f"model.layers.2.mlp.experts.{base_layer_prefix}routed_experts.w13_weight"
|
||||||
|
)
|
||||||
|
params_dict = {fused_name: experts.w13_weight}
|
||||||
|
loaded_params: set[str] = set()
|
||||||
|
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
|
||||||
|
|
||||||
|
assert GptOssModel._get_streamed_expert_info(mapped_name, params_dict) == (
|
||||||
|
7,
|
||||||
|
fused_name,
|
||||||
|
"gpt_oss_w13",
|
||||||
|
)
|
||||||
|
assert GptOssModel._try_load_streamed_expert(
|
||||||
|
mapped_name, loaded_weight, params_dict, loaded_params
|
||||||
|
)
|
||||||
|
assert loaded_params == {fused_name}
|
||||||
|
assert torch.equal(experts.w13_weight[1], loaded_weight[:, :4].t())
|
||||||
|
assert torch.count_nonzero(experts.w13_weight[0]) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_streamed_expert_skips_non_local_expert() -> None:
|
||||||
|
experts = _TestExperts("w2_bias", (2, 3))
|
||||||
|
|
||||||
|
assert not _load(experts, "w2_bias", torch.ones(3), "gpt_oss_w2", expert_id=6)
|
||||||
|
assert torch.count_nonzero(experts.w2_bias) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("param_name", ["w13_weight", "w2_weight"])
|
||||||
|
def test_streamed_unquantized_expert_tp_slice(param_name: str) -> None:
|
||||||
|
if param_name == "w13_weight":
|
||||||
|
experts = _TestExperts(param_name, (2, 4, 3), tp_rank=1)
|
||||||
|
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
|
||||||
|
shard_id = "gpt_oss_w13"
|
||||||
|
expected = loaded_weight[:, 4:].t()
|
||||||
|
else:
|
||||||
|
experts = _TestExperts(param_name, (2, 3, 2), tp_rank=1)
|
||||||
|
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
|
||||||
|
shard_id = "gpt_oss_w2"
|
||||||
|
expected = loaded_weight[2:].t()
|
||||||
|
|
||||||
|
assert _load(experts, param_name, loaded_weight, shard_id)
|
||||||
|
assert torch.equal(getattr(experts, param_name)[0], expected)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||||
|
def test_streamed_down_bias_owned_by_tp_rank_zero(tp_rank: int) -> None:
|
||||||
|
experts = _TestExperts("w2_bias", (2, 3), tp_rank=tp_rank)
|
||||||
|
loaded_weight = torch.arange(1, 4, dtype=torch.float32)
|
||||||
|
|
||||||
|
assert _load(experts, "w2_bias", loaded_weight, "gpt_oss_w2")
|
||||||
|
expected = loaded_weight if tp_rank == 0 else torch.zeros_like(loaded_weight)
|
||||||
|
assert torch.equal(experts.w2_bias[0], expected)
|
||||||
|
|
||||||
|
|
||||||
|
def test_streamed_gate_up_bias_tp_slice() -> None:
|
||||||
|
experts = _TestExperts("w13_bias", (2, 4), tp_rank=1)
|
||||||
|
loaded_weight = torch.arange(8, dtype=torch.float32)
|
||||||
|
|
||||||
|
assert _load(experts, "w13_bias", loaded_weight, "gpt_oss_w13")
|
||||||
|
assert torch.equal(experts.w13_bias[0], loaded_weight[4:])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("case", ["mxfp4", "nvfp4"])
|
||||||
|
def test_streamed_quantized_expert_layout(case: str) -> None:
|
||||||
|
if case == "mxfp4":
|
||||||
|
experts = _TestExperts(
|
||||||
|
"w13_weight",
|
||||||
|
(2, 4, 4),
|
||||||
|
tp_rank=1,
|
||||||
|
weight_dtype="gpt_oss_mxfp4",
|
||||||
|
dtype=torch.uint8,
|
||||||
|
)
|
||||||
|
loaded_weight = torch.arange(32, dtype=torch.uint8).reshape(8, 2, 2)
|
||||||
|
expected = loaded_weight.reshape(8, 4)[4:]
|
||||||
|
else:
|
||||||
|
experts = _TestExperts(
|
||||||
|
"w13_weight",
|
||||||
|
(2, 4, 2),
|
||||||
|
tp_rank=1,
|
||||||
|
quant_method_name="Nvfp4OnlineMoEMethod",
|
||||||
|
dtype=torch.uint8,
|
||||||
|
)
|
||||||
|
loaded_weight = torch.arange(16, dtype=torch.uint8).reshape(8, 2)
|
||||||
|
expected = loaded_weight[4:]
|
||||||
|
|
||||||
|
assert _load(experts, "w13_weight", loaded_weight, "gpt_oss_w13")
|
||||||
|
assert torch.equal(experts.w13_weight[0], expected)
|
||||||
@@ -218,6 +218,21 @@ def test_reload_lifecycle():
|
|||||||
assert tensor.__dict__ == materialized_tensor.__dict__
|
assert tensor.__dict__ == materialized_tensor.__dict__
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_layer_replaces_postprocessed_tensor_attribute():
|
||||||
|
layer = torch.nn.Linear(2, 3, bias=False)
|
||||||
|
info = LayerReloadingInfo(
|
||||||
|
restore_metadata=capture_layer_to_meta(layer),
|
||||||
|
restore_device=torch.device("cpu"),
|
||||||
|
)
|
||||||
|
del layer.weight
|
||||||
|
layer.weight = torch.empty(3, 2)
|
||||||
|
|
||||||
|
restore_layer_on_meta(layer, info)
|
||||||
|
|
||||||
|
assert isinstance(layer.weight, torch.nn.Parameter)
|
||||||
|
assert layer.weight.is_meta
|
||||||
|
|
||||||
|
|
||||||
def test_materialize_layer_preserves_non_meta_tensors():
|
def test_materialize_layer_preserves_non_meta_tensors():
|
||||||
"""Ensure that materialize_layer does not overwrite non meta tensors."""
|
"""Ensure that materialize_layer does not overwrite non meta tensors."""
|
||||||
layer = torch.nn.Linear(2, 3, bias=True)
|
layer = torch.nn.Linear(2, 3, bias=True)
|
||||||
|
|||||||
@@ -124,11 +124,14 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
|
|||||||
return
|
return
|
||||||
|
|
||||||
non_persistent = set(layer._non_persistent_buffers_set)
|
non_persistent = set(layer._non_persistent_buffers_set)
|
||||||
for name in get_layer_tensors(layer):
|
restore_params, restore_buffers = info.restore_metadata
|
||||||
if name not in SKIP_TENSORS:
|
tensor_names = (
|
||||||
|
get_layer_tensors(layer).keys() | restore_params.keys() | restore_buffers.keys()
|
||||||
|
)
|
||||||
|
for name in tensor_names:
|
||||||
|
if name not in SKIP_TENSORS and hasattr(layer, name):
|
||||||
delattr(layer, name)
|
delattr(layer, name)
|
||||||
|
|
||||||
restore_params, restore_buffers = info.restore_metadata
|
|
||||||
for name, param in restore_params.items():
|
for name, param in restore_params.items():
|
||||||
if name not in SKIP_TENSORS:
|
if name not in SKIP_TENSORS:
|
||||||
param = restore_layer_refs(param, layer)
|
param = restore_layer_refs(param, layer)
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe import (
|
|||||||
fused_moe_make_expert_params_mapping,
|
fused_moe_make_expert_params_mapping,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||||
|
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -68,6 +69,15 @@ from .utils import (
|
|||||||
maybe_prefix,
|
maybe_prefix,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
|
||||||
|
"w13_weight": "gpt_oss_w13",
|
||||||
|
"w2_weight": "gpt_oss_w2",
|
||||||
|
"w13_bias": "gpt_oss_w13",
|
||||||
|
"w2_bias": "gpt_oss_w2",
|
||||||
|
"w13_weight_scale": "gpt_oss_w13",
|
||||||
|
"w2_weight_scale": "gpt_oss_w2",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class OAIAttention(nn.Module):
|
class OAIAttention(nn.Module):
|
||||||
# Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves);
|
# Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves);
|
||||||
@@ -179,6 +189,132 @@ class OAIAttention(nn.Module):
|
|||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
class GptOssRoutedExperts(RoutedExperts):
|
||||||
|
"""Load one GPT-OSS expert at a time without assembling the global stack."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _narrow_for_rank(
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
dim: int,
|
||||||
|
rank: int,
|
||||||
|
size: int,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
start = rank * size
|
||||||
|
available = loaded_weight.shape[dim] - start
|
||||||
|
return loaded_weight.narrow(dim, start, min(size, max(available, 0)))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _copy_to_expert(expert_data: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||||
|
if expert_data.numel() == loaded_weight.numel() == 1:
|
||||||
|
expert_data.copy_(loaded_weight.reshape(()))
|
||||||
|
return
|
||||||
|
while loaded_weight.ndim > expert_data.ndim and loaded_weight.shape[-1] == 1:
|
||||||
|
loaded_weight = loaded_weight.squeeze(-1)
|
||||||
|
slices = tuple(slice(0, size) for size in loaded_weight.shape)
|
||||||
|
expert_data[slices].copy_(loaded_weight)
|
||||||
|
|
||||||
|
def _load_expert_bias(
|
||||||
|
self,
|
||||||
|
expert_data: torch.Tensor,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
shard_id: str,
|
||||||
|
) -> None:
|
||||||
|
tp_rank = self.moe_config.moe_parallel_config.tp_rank
|
||||||
|
if shard_id == "gpt_oss_w13":
|
||||||
|
loaded_weight = self._narrow_for_rank(
|
||||||
|
loaded_weight, 0, tp_rank, expert_data.shape[0]
|
||||||
|
)
|
||||||
|
elif tp_rank != 0:
|
||||||
|
loaded_weight = torch.zeros_like(loaded_weight)
|
||||||
|
self._copy_to_expert(expert_data, loaded_weight)
|
||||||
|
|
||||||
|
def _load_unquantized_expert(
|
||||||
|
self,
|
||||||
|
expert_data: torch.Tensor,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
shard_id: str,
|
||||||
|
) -> None:
|
||||||
|
tp_rank = self.moe_config.moe_parallel_config.tp_rank
|
||||||
|
if shard_id == "gpt_oss_w13":
|
||||||
|
loaded_weight = self._narrow_for_rank(
|
||||||
|
loaded_weight, 1, tp_rank, expert_data.shape[0]
|
||||||
|
)
|
||||||
|
loaded_weight = loaded_weight.t().contiguous()
|
||||||
|
else:
|
||||||
|
loaded_weight = self._narrow_for_rank(
|
||||||
|
loaded_weight, 0, tp_rank, expert_data.shape[1]
|
||||||
|
)
|
||||||
|
loaded_weight = loaded_weight.t().contiguous()
|
||||||
|
self._copy_to_expert(expert_data, loaded_weight)
|
||||||
|
|
||||||
|
def _load_packed_expert(
|
||||||
|
self,
|
||||||
|
expert_data: torch.Tensor,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
weight_name: str,
|
||||||
|
shard_id: str,
|
||||||
|
) -> None:
|
||||||
|
tp_rank = self.moe_config.moe_parallel_config.tp_rank
|
||||||
|
is_w13 = shard_id == "gpt_oss_w13"
|
||||||
|
is_weight = weight_name.endswith("_weight")
|
||||||
|
is_partitioned_scale = weight_name.endswith("_weight_scale")
|
||||||
|
|
||||||
|
if is_weight:
|
||||||
|
loaded_weight = loaded_weight.reshape(loaded_weight.shape[0], -1)
|
||||||
|
|
||||||
|
if is_w13 and (is_weight or is_partitioned_scale):
|
||||||
|
loaded_weight = self._narrow_for_rank(
|
||||||
|
loaded_weight, 0, tp_rank, expert_data.shape[0]
|
||||||
|
)
|
||||||
|
elif not is_w13 and (is_weight or is_partitioned_scale):
|
||||||
|
loaded_weight = self._narrow_for_rank(
|
||||||
|
loaded_weight, -1, tp_rank, expert_data.shape[-1]
|
||||||
|
)
|
||||||
|
|
||||||
|
self._copy_to_expert(expert_data, loaded_weight)
|
||||||
|
|
||||||
|
def weight_loader(
|
||||||
|
self,
|
||||||
|
param: torch.nn.Parameter,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
weight_name: str,
|
||||||
|
shard_id: str,
|
||||||
|
expert_id: int,
|
||||||
|
return_success: bool = False,
|
||||||
|
) -> bool | None:
|
||||||
|
if shard_id not in ("gpt_oss_w13", "gpt_oss_w2"):
|
||||||
|
return super().weight_loader(
|
||||||
|
param,
|
||||||
|
loaded_weight,
|
||||||
|
weight_name,
|
||||||
|
shard_id,
|
||||||
|
expert_id,
|
||||||
|
return_success,
|
||||||
|
)
|
||||||
|
|
||||||
|
expert_id = self._map_global_expert_id_to_local_expert_id(expert_id)
|
||||||
|
if expert_id == -1:
|
||||||
|
return False if return_success else None
|
||||||
|
|
||||||
|
expert_data = param.data[expert_id]
|
||||||
|
weight_dtype = getattr(self.quant_method, "weight_dtype", "")
|
||||||
|
quant_method_name = self.quant_method.__class__.__name__
|
||||||
|
|
||||||
|
if weight_name.endswith("_bias"):
|
||||||
|
self._load_expert_bias(expert_data, loaded_weight, shard_id)
|
||||||
|
elif weight_dtype in (
|
||||||
|
"gpt_oss_mxfp4",
|
||||||
|
"mxfp4",
|
||||||
|
) or quant_method_name in (
|
||||||
|
"CompressedTensorsW4A4Nvfp4MoEMethod",
|
||||||
|
"Nvfp4OnlineMoEMethod",
|
||||||
|
):
|
||||||
|
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
|
||||||
|
else:
|
||||||
|
self._load_unquantized_expert(expert_data, loaded_weight, shard_id)
|
||||||
|
return True if return_success else None
|
||||||
|
|
||||||
|
|
||||||
class MLPBlock(torch.nn.Module):
|
class MLPBlock(torch.nn.Module):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -223,6 +359,7 @@ class MLPBlock(torch.nn.Module):
|
|||||||
has_bias=True,
|
has_bias=True,
|
||||||
activation="swigluoai",
|
activation="swigluoai",
|
||||||
is_sequence_parallel=self.is_sequence_parallel,
|
is_sequence_parallel=self.is_sequence_parallel,
|
||||||
|
routed_experts_cls=GptOssRoutedExperts,
|
||||||
)
|
)
|
||||||
|
|
||||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||||
@@ -416,6 +553,9 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
|||||||
if is_pp_missing_parameter(name, self):
|
if is_pp_missing_parameter(name, self):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if self._try_load_streamed_expert(name, weight, params_dict, loaded_params):
|
||||||
|
continue
|
||||||
|
|
||||||
if ".w13_weight_scale" in name:
|
if ".w13_weight_scale" in name:
|
||||||
# Handle MLP gate and up projection weights scale
|
# Handle MLP gate and up projection weights scale
|
||||||
if use_ep:
|
if use_ep:
|
||||||
@@ -634,6 +774,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
|||||||
tp_rank_start = tp_rank * per_rank_intermediate_size
|
tp_rank_start = tp_rank * per_rank_intermediate_size
|
||||||
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
|
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
|
||||||
expert_params_mapping = self.get_expert_mapping()
|
expert_params_mapping = self.get_expert_mapping()
|
||||||
|
# Streamed per-expert reload is intentionally unsupported for Quark.
|
||||||
for name, loaded_weight in weights:
|
for name, loaded_weight in weights:
|
||||||
if is_pp_missing_parameter(name, self):
|
if is_pp_missing_parameter(name, self):
|
||||||
continue
|
continue
|
||||||
@@ -1018,6 +1159,9 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
|||||||
if is_pp_missing_parameter(name, self):
|
if is_pp_missing_parameter(name, self):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if self._try_load_streamed_expert(name, weight, params_dict, loaded_params):
|
||||||
|
continue
|
||||||
|
|
||||||
if ".w13_weight" in name:
|
if ".w13_weight" in name:
|
||||||
# Handle MLP gate and up projection weights
|
# Handle MLP gate and up projection weights
|
||||||
# Extract gate and up projection parts
|
# Extract gate and up projection parts
|
||||||
@@ -1096,6 +1240,59 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
|||||||
loaded_params.add(name)
|
loaded_params.add(name)
|
||||||
return loaded_params
|
return loaded_params
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_streamed_expert_info(
|
||||||
|
name: str,
|
||||||
|
params_dict: dict[str, torch.nn.Parameter],
|
||||||
|
) -> tuple[int, str, str] | None:
|
||||||
|
"""Parse ``...experts.<expert_id>.<fused_param>`` checkpoint keys."""
|
||||||
|
if ".mlp.experts." not in name:
|
||||||
|
return None
|
||||||
|
suffix = name.rsplit(".", 1)[-1]
|
||||||
|
shard_id = _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD.get(suffix)
|
||||||
|
if shard_id is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
prefix, expert_suffix = name.split(".mlp.experts.", maxsplit=1)
|
||||||
|
expert_id_str, separator, param_suffix = expert_suffix.partition(".")
|
||||||
|
if not separator or not expert_id_str.isdigit():
|
||||||
|
return None
|
||||||
|
expert_id = int(expert_id_str)
|
||||||
|
for base_layer_prefix in ("", "base_layer."):
|
||||||
|
fused_name = (
|
||||||
|
f"{prefix}.mlp.experts.{base_layer_prefix}routed_experts.{param_suffix}"
|
||||||
|
)
|
||||||
|
if fused_name in params_dict:
|
||||||
|
return expert_id, fused_name, shard_id
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _try_load_streamed_expert(
|
||||||
|
cls,
|
||||||
|
name: str,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
params_dict: dict[str, torch.nn.Parameter],
|
||||||
|
loaded_params: set[str],
|
||||||
|
) -> bool:
|
||||||
|
expert_info = cls._get_streamed_expert_info(name, params_dict)
|
||||||
|
if expert_info is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
expert_id, fused_name, shard_id = expert_info
|
||||||
|
param = params_dict[fused_name]
|
||||||
|
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||||
|
success = weight_loader(
|
||||||
|
param,
|
||||||
|
loaded_weight,
|
||||||
|
weight_name=fused_name,
|
||||||
|
shard_id=shard_id,
|
||||||
|
expert_id=expert_id,
|
||||||
|
return_success=True,
|
||||||
|
)
|
||||||
|
if success:
|
||||||
|
loaded_params.add(fused_name)
|
||||||
|
return True
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||||
stacked_params_mapping = [
|
stacked_params_mapping = [
|
||||||
# (param_name, shard_name, shard_id)
|
# (param_name, shard_name, shard_id)
|
||||||
|
|||||||
Reference in New Issue
Block a user