Add routed expert loading for gpt-oss (#52209)

Signed-off-by: Wyett <wyettzeng@gmail.com>
Signed-off-by: wyett <wyettzeng@gmail.com>
This commit is contained in:
Wyett
2026-08-23 17:59:08 +08:00
committed by GitHub
parent e25c586b90
commit b26039b09f
4 changed files with 370 additions and 3 deletions
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.models.gpt_oss import (
GptOssForCausalLM,
GptOssModel,
GptOssRoutedExperts,
)
class _TestExperts(GptOssRoutedExperts):
def __init__(
self,
param_name: str,
shape: tuple[int, ...],
*,
tp_rank: int = 0,
quant_method_name: str = "TestQuantMethod",
weight_dtype: str = "",
dtype: torch.dtype = torch.float32,
) -> None:
torch.nn.Module.__init__(self)
self.moe_config = SimpleNamespace(
moe_parallel_config=SimpleNamespace(tp_size=2, tp_rank=tp_rank)
)
self.quant_method = type(
quant_method_name, (), {"weight_dtype": weight_dtype}
)()
param = torch.nn.Parameter(torch.zeros(shape, dtype=dtype), requires_grad=False)
param.weight_loader = self.weight_loader
self.register_parameter(param_name, param)
@staticmethod
def _map_global_expert_id_to_local_expert_id(expert_id: int) -> int:
return {3: 0, 7: 1}.get(expert_id, -1)
def _load(
experts: _TestExperts,
param_name: str,
loaded_weight: torch.Tensor,
shard_id: str,
expert_id: int = 3,
) -> bool:
param = getattr(experts, param_name)
return param.weight_loader(
param,
loaded_weight,
weight_name=f"model.layers.0.mlp.experts.routed_experts.{param_name}",
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
@pytest.mark.parametrize("base_layer_prefix", ["", "base_layer."])
def test_streamed_expert_resolves_wrapped_parameter(base_layer_prefix: str) -> None:
experts = _TestExperts("w13_weight", (2, 4, 3))
checkpoint_name = "model.layers.2.mlp.experts.7.gate_up_proj"
mapped_name = GptOssForCausalLM.hf_to_vllm_mapper.apply_list([checkpoint_name])[0]
fused_name = (
f"model.layers.2.mlp.experts.{base_layer_prefix}routed_experts.w13_weight"
)
params_dict = {fused_name: experts.w13_weight}
loaded_params: set[str] = set()
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
assert GptOssModel._get_streamed_expert_info(mapped_name, params_dict) == (
7,
fused_name,
"gpt_oss_w13",
)
assert GptOssModel._try_load_streamed_expert(
mapped_name, loaded_weight, params_dict, loaded_params
)
assert loaded_params == {fused_name}
assert torch.equal(experts.w13_weight[1], loaded_weight[:, :4].t())
assert torch.count_nonzero(experts.w13_weight[0]) == 0
def test_streamed_expert_skips_non_local_expert() -> None:
experts = _TestExperts("w2_bias", (2, 3))
assert not _load(experts, "w2_bias", torch.ones(3), "gpt_oss_w2", expert_id=6)
assert torch.count_nonzero(experts.w2_bias) == 0
@pytest.mark.parametrize("param_name", ["w13_weight", "w2_weight"])
def test_streamed_unquantized_expert_tp_slice(param_name: str) -> None:
if param_name == "w13_weight":
experts = _TestExperts(param_name, (2, 4, 3), tp_rank=1)
loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8)
shard_id = "gpt_oss_w13"
expected = loaded_weight[:, 4:].t()
else:
experts = _TestExperts(param_name, (2, 3, 2), tp_rank=1)
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
shard_id = "gpt_oss_w2"
expected = loaded_weight[2:].t()
assert _load(experts, param_name, loaded_weight, shard_id)
assert torch.equal(getattr(experts, param_name)[0], expected)
@pytest.mark.parametrize("tp_rank", [0, 1])
def test_streamed_down_bias_owned_by_tp_rank_zero(tp_rank: int) -> None:
experts = _TestExperts("w2_bias", (2, 3), tp_rank=tp_rank)
loaded_weight = torch.arange(1, 4, dtype=torch.float32)
assert _load(experts, "w2_bias", loaded_weight, "gpt_oss_w2")
expected = loaded_weight if tp_rank == 0 else torch.zeros_like(loaded_weight)
assert torch.equal(experts.w2_bias[0], expected)
def test_streamed_gate_up_bias_tp_slice() -> None:
experts = _TestExperts("w13_bias", (2, 4), tp_rank=1)
loaded_weight = torch.arange(8, dtype=torch.float32)
assert _load(experts, "w13_bias", loaded_weight, "gpt_oss_w13")
assert torch.equal(experts.w13_bias[0], loaded_weight[4:])
@pytest.mark.parametrize("case", ["mxfp4", "nvfp4"])
def test_streamed_quantized_expert_layout(case: str) -> None:
if case == "mxfp4":
experts = _TestExperts(
"w13_weight",
(2, 4, 4),
tp_rank=1,
weight_dtype="gpt_oss_mxfp4",
dtype=torch.uint8,
)
loaded_weight = torch.arange(32, dtype=torch.uint8).reshape(8, 2, 2)
expected = loaded_weight.reshape(8, 4)[4:]
else:
experts = _TestExperts(
"w13_weight",
(2, 4, 2),
tp_rank=1,
quant_method_name="Nvfp4OnlineMoEMethod",
dtype=torch.uint8,
)
loaded_weight = torch.arange(16, dtype=torch.uint8).reshape(8, 2)
expected = loaded_weight[4:]
assert _load(experts, "w13_weight", loaded_weight, "gpt_oss_w13")
assert torch.equal(experts.w13_weight[0], expected)
@@ -218,6 +218,21 @@ def test_reload_lifecycle():
assert tensor.__dict__ == materialized_tensor.__dict__ assert tensor.__dict__ == materialized_tensor.__dict__
def test_restore_layer_replaces_postprocessed_tensor_attribute():
layer = torch.nn.Linear(2, 3, bias=False)
info = LayerReloadingInfo(
restore_metadata=capture_layer_to_meta(layer),
restore_device=torch.device("cpu"),
)
del layer.weight
layer.weight = torch.empty(3, 2)
restore_layer_on_meta(layer, info)
assert isinstance(layer.weight, torch.nn.Parameter)
assert layer.weight.is_meta
def test_materialize_layer_preserves_non_meta_tensors(): def test_materialize_layer_preserves_non_meta_tensors():
"""Ensure that materialize_layer does not overwrite non meta tensors.""" """Ensure that materialize_layer does not overwrite non meta tensors."""
layer = torch.nn.Linear(2, 3, bias=True) layer = torch.nn.Linear(2, 3, bias=True)
@@ -124,11 +124,14 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
return return
non_persistent = set(layer._non_persistent_buffers_set) non_persistent = set(layer._non_persistent_buffers_set)
for name in get_layer_tensors(layer): restore_params, restore_buffers = info.restore_metadata
if name not in SKIP_TENSORS: tensor_names = (
get_layer_tensors(layer).keys() | restore_params.keys() | restore_buffers.keys()
)
for name in tensor_names:
if name not in SKIP_TENSORS and hasattr(layer, name):
delattr(layer, name) delattr(layer, name)
restore_params, restore_buffers = info.restore_metadata
for name, param in restore_params.items(): for name, param in restore_params.items():
if name not in SKIP_TENSORS: if name not in SKIP_TENSORS:
param = restore_layer_refs(param, layer) param = restore_layer_refs(param, layer)
+197
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping, fused_moe_make_expert_params_mapping,
) )
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.linear import (
QKVParallelLinear, QKVParallelLinear,
@@ -68,6 +69,15 @@ from .utils import (
maybe_prefix, maybe_prefix,
) )
_GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
"w13_weight": "gpt_oss_w13",
"w2_weight": "gpt_oss_w2",
"w13_bias": "gpt_oss_w13",
"w2_bias": "gpt_oss_w2",
"w13_weight_scale": "gpt_oss_w13",
"w2_weight_scale": "gpt_oss_w2",
}
class OAIAttention(nn.Module): class OAIAttention(nn.Module):
# Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves); # Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves);
@@ -179,6 +189,132 @@ class OAIAttention(nn.Module):
return output return output
class GptOssRoutedExperts(RoutedExperts):
"""Load one GPT-OSS expert at a time without assembling the global stack."""
@staticmethod
def _narrow_for_rank(
loaded_weight: torch.Tensor,
dim: int,
rank: int,
size: int,
) -> torch.Tensor:
start = rank * size
available = loaded_weight.shape[dim] - start
return loaded_weight.narrow(dim, start, min(size, max(available, 0)))
@staticmethod
def _copy_to_expert(expert_data: torch.Tensor, loaded_weight: torch.Tensor) -> None:
if expert_data.numel() == loaded_weight.numel() == 1:
expert_data.copy_(loaded_weight.reshape(()))
return
while loaded_weight.ndim > expert_data.ndim and loaded_weight.shape[-1] == 1:
loaded_weight = loaded_weight.squeeze(-1)
slices = tuple(slice(0, size) for size in loaded_weight.shape)
expert_data[slices].copy_(loaded_weight)
def _load_expert_bias(
self,
expert_data: torch.Tensor,
loaded_weight: torch.Tensor,
shard_id: str,
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
if shard_id == "gpt_oss_w13":
loaded_weight = self._narrow_for_rank(
loaded_weight, 0, tp_rank, expert_data.shape[0]
)
elif tp_rank != 0:
loaded_weight = torch.zeros_like(loaded_weight)
self._copy_to_expert(expert_data, loaded_weight)
def _load_unquantized_expert(
self,
expert_data: torch.Tensor,
loaded_weight: torch.Tensor,
shard_id: str,
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
if shard_id == "gpt_oss_w13":
loaded_weight = self._narrow_for_rank(
loaded_weight, 1, tp_rank, expert_data.shape[0]
)
loaded_weight = loaded_weight.t().contiguous()
else:
loaded_weight = self._narrow_for_rank(
loaded_weight, 0, tp_rank, expert_data.shape[1]
)
loaded_weight = loaded_weight.t().contiguous()
self._copy_to_expert(expert_data, loaded_weight)
def _load_packed_expert(
self,
expert_data: torch.Tensor,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
is_w13 = shard_id == "gpt_oss_w13"
is_weight = weight_name.endswith("_weight")
is_partitioned_scale = weight_name.endswith("_weight_scale")
if is_weight:
loaded_weight = loaded_weight.reshape(loaded_weight.shape[0], -1)
if is_w13 and (is_weight or is_partitioned_scale):
loaded_weight = self._narrow_for_rank(
loaded_weight, 0, tp_rank, expert_data.shape[0]
)
elif not is_w13 and (is_weight or is_partitioned_scale):
loaded_weight = self._narrow_for_rank(
loaded_weight, -1, tp_rank, expert_data.shape[-1]
)
self._copy_to_expert(expert_data, loaded_weight)
def weight_loader(
self,
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
if shard_id not in ("gpt_oss_w13", "gpt_oss_w2"):
return super().weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success,
)
expert_id = self._map_global_expert_id_to_local_expert_id(expert_id)
if expert_id == -1:
return False if return_success else None
expert_data = param.data[expert_id]
weight_dtype = getattr(self.quant_method, "weight_dtype", "")
quant_method_name = self.quant_method.__class__.__name__
if weight_name.endswith("_bias"):
self._load_expert_bias(expert_data, loaded_weight, shard_id)
elif weight_dtype in (
"gpt_oss_mxfp4",
"mxfp4",
) or quant_method_name in (
"CompressedTensorsW4A4Nvfp4MoEMethod",
"Nvfp4OnlineMoEMethod",
):
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
else:
self._load_unquantized_expert(expert_data, loaded_weight, shard_id)
return True if return_success else None
class MLPBlock(torch.nn.Module): class MLPBlock(torch.nn.Module):
def __init__( def __init__(
self, self,
@@ -223,6 +359,7 @@ class MLPBlock(torch.nn.Module):
has_bias=True, has_bias=True,
activation="swigluoai", activation="swigluoai",
is_sequence_parallel=self.is_sequence_parallel, is_sequence_parallel=self.is_sequence_parallel,
routed_experts_cls=GptOssRoutedExperts,
) )
def forward(self, x: torch.Tensor) -> torch.Tensor: def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -416,6 +553,9 @@ class GptOssModel(nn.Module, EagleModelMixin):
if is_pp_missing_parameter(name, self): if is_pp_missing_parameter(name, self):
continue continue
if self._try_load_streamed_expert(name, weight, params_dict, loaded_params):
continue
if ".w13_weight_scale" in name: if ".w13_weight_scale" in name:
# Handle MLP gate and up projection weights scale # Handle MLP gate and up projection weights scale
if use_ep: if use_ep:
@@ -634,6 +774,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
tp_rank_start = tp_rank * per_rank_intermediate_size tp_rank_start = tp_rank * per_rank_intermediate_size
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size) tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
expert_params_mapping = self.get_expert_mapping() expert_params_mapping = self.get_expert_mapping()
# Streamed per-expert reload is intentionally unsupported for Quark.
for name, loaded_weight in weights: for name, loaded_weight in weights:
if is_pp_missing_parameter(name, self): if is_pp_missing_parameter(name, self):
continue continue
@@ -1018,6 +1159,9 @@ class GptOssModel(nn.Module, EagleModelMixin):
if is_pp_missing_parameter(name, self): if is_pp_missing_parameter(name, self):
continue continue
if self._try_load_streamed_expert(name, weight, params_dict, loaded_params):
continue
if ".w13_weight" in name: if ".w13_weight" in name:
# Handle MLP gate and up projection weights # Handle MLP gate and up projection weights
# Extract gate and up projection parts # Extract gate and up projection parts
@@ -1096,6 +1240,59 @@ class GptOssModel(nn.Module, EagleModelMixin):
loaded_params.add(name) loaded_params.add(name)
return loaded_params return loaded_params
@staticmethod
def _get_streamed_expert_info(
name: str,
params_dict: dict[str, torch.nn.Parameter],
) -> tuple[int, str, str] | None:
"""Parse ``...experts.<expert_id>.<fused_param>`` checkpoint keys."""
if ".mlp.experts." not in name:
return None
suffix = name.rsplit(".", 1)[-1]
shard_id = _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD.get(suffix)
if shard_id is None:
return None
prefix, expert_suffix = name.split(".mlp.experts.", maxsplit=1)
expert_id_str, separator, param_suffix = expert_suffix.partition(".")
if not separator or not expert_id_str.isdigit():
return None
expert_id = int(expert_id_str)
for base_layer_prefix in ("", "base_layer."):
fused_name = (
f"{prefix}.mlp.experts.{base_layer_prefix}routed_experts.{param_suffix}"
)
if fused_name in params_dict:
return expert_id, fused_name, shard_id
return None
@classmethod
def _try_load_streamed_expert(
cls,
name: str,
loaded_weight: torch.Tensor,
params_dict: dict[str, torch.nn.Parameter],
loaded_params: set[str],
) -> bool:
expert_info = cls._get_streamed_expert_info(name, params_dict)
if expert_info is None:
return False
expert_id, fused_name, shard_id = expert_info
param = params_dict[fused_name]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
success = weight_loader(
param,
loaded_weight,
weight_name=fused_name,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_params.add(fused_name)
return True
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [ stacked_params_mapping = [
# (param_name, shard_name, shard_id) # (param_name, shard_name, shard_id)