Add routed expert loading for gpt-oss (#52209)

Signed-off-by: Wyett <wyettzeng@gmail.com>
Signed-off-by: wyett <wyettzeng@gmail.com>
This commit is contained in:
Wyett
2026-08-23 17:59:08 +08:00
committed by GitHub
parent e25c586b90
commit b26039b09f
4 changed files with 370 additions and 3 deletions
@@ -124,11 +124,14 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
return
non_persistent = set(layer._non_persistent_buffers_set)
for name in get_layer_tensors(layer):
if name not in SKIP_TENSORS:
restore_params, restore_buffers = info.restore_metadata
tensor_names = (
get_layer_tensors(layer).keys() | restore_params.keys() | restore_buffers.keys()
)
for name in tensor_names:
if name not in SKIP_TENSORS and hasattr(layer, name):
delattr(layer, name)
restore_params, restore_buffers = info.restore_metadata
for name, param in restore_params.items():
if name not in SKIP_TENSORS:
param = restore_layer_refs(param, layer)
+197
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -68,6 +69,15 @@ from .utils import (
maybe_prefix,
)
_GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
"w13_weight": "gpt_oss_w13",
"w2_weight": "gpt_oss_w2",
"w13_bias": "gpt_oss_w13",
"w2_bias": "gpt_oss_w2",
"w13_weight_scale": "gpt_oss_w13",
"w2_weight_scale": "gpt_oss_w2",
}
class OAIAttention(nn.Module):
# Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves);
@@ -179,6 +189,132 @@ class OAIAttention(nn.Module):
return output
class GptOssRoutedExperts(RoutedExperts):
"""Load one GPT-OSS expert at a time without assembling the global stack."""
@staticmethod
def _narrow_for_rank(
loaded_weight: torch.Tensor,
dim: int,
rank: int,
size: int,
) -> torch.Tensor:
start = rank * size
available = loaded_weight.shape[dim] - start
return loaded_weight.narrow(dim, start, min(size, max(available, 0)))
@staticmethod
def _copy_to_expert(expert_data: torch.Tensor, loaded_weight: torch.Tensor) -> None:
if expert_data.numel() == loaded_weight.numel() == 1:
expert_data.copy_(loaded_weight.reshape(()))
return
while loaded_weight.ndim > expert_data.ndim and loaded_weight.shape[-1] == 1:
loaded_weight = loaded_weight.squeeze(-1)
slices = tuple(slice(0, size) for size in loaded_weight.shape)
expert_data[slices].copy_(loaded_weight)
def _load_expert_bias(
self,
expert_data: torch.Tensor,
loaded_weight: torch.Tensor,
shard_id: str,
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
if shard_id == "gpt_oss_w13":
loaded_weight = self._narrow_for_rank(
loaded_weight, 0, tp_rank, expert_data.shape[0]
)
elif tp_rank != 0:
loaded_weight = torch.zeros_like(loaded_weight)
self._copy_to_expert(expert_data, loaded_weight)
def _load_unquantized_expert(
self,
expert_data: torch.Tensor,
loaded_weight: torch.Tensor,
shard_id: str,
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
if shard_id == "gpt_oss_w13":
loaded_weight = self._narrow_for_rank(
loaded_weight, 1, tp_rank, expert_data.shape[0]
)
loaded_weight = loaded_weight.t().contiguous()
else:
loaded_weight = self._narrow_for_rank(
loaded_weight, 0, tp_rank, expert_data.shape[1]
)
loaded_weight = loaded_weight.t().contiguous()
self._copy_to_expert(expert_data, loaded_weight)
def _load_packed_expert(
self,
expert_data: torch.Tensor,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
is_w13 = shard_id == "gpt_oss_w13"
is_weight = weight_name.endswith("_weight")
is_partitioned_scale = weight_name.endswith("_weight_scale")
if is_weight:
loaded_weight = loaded_weight.reshape(loaded_weight.shape[0], -1)
if is_w13 and (is_weight or is_partitioned_scale):
loaded_weight = self._narrow_for_rank(
loaded_weight, 0, tp_rank, expert_data.shape[0]
)
elif not is_w13 and (is_weight or is_partitioned_scale):
loaded_weight = self._narrow_for_rank(
loaded_weight, -1, tp_rank, expert_data.shape[-1]
)
self._copy_to_expert(expert_data, loaded_weight)
def weight_loader(
self,
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
if shard_id not in ("gpt_oss_w13", "gpt_oss_w2"):
return super().weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success,
)
expert_id = self._map_global_expert_id_to_local_expert_id(expert_id)
if expert_id == -1:
return False if return_success else None
expert_data = param.data[expert_id]
weight_dtype = getattr(self.quant_method, "weight_dtype", "")
quant_method_name = self.quant_method.__class__.__name__
if weight_name.endswith("_bias"):
self._load_expert_bias(expert_data, loaded_weight, shard_id)
elif weight_dtype in (
"gpt_oss_mxfp4",
"mxfp4",
) or quant_method_name in (
"CompressedTensorsW4A4Nvfp4MoEMethod",
"Nvfp4OnlineMoEMethod",
):
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
else:
self._load_unquantized_expert(expert_data, loaded_weight, shard_id)
return True if return_success else None
class MLPBlock(torch.nn.Module):
def __init__(
self,
@@ -223,6 +359,7 @@ class MLPBlock(torch.nn.Module):
has_bias=True,
activation="swigluoai",
is_sequence_parallel=self.is_sequence_parallel,
routed_experts_cls=GptOssRoutedExperts,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -416,6 +553,9 @@ class GptOssModel(nn.Module, EagleModelMixin):
if is_pp_missing_parameter(name, self):
continue
if self._try_load_streamed_expert(name, weight, params_dict, loaded_params):
continue
if ".w13_weight_scale" in name:
# Handle MLP gate and up projection weights scale
if use_ep:
@@ -634,6 +774,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
tp_rank_start = tp_rank * per_rank_intermediate_size
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
expert_params_mapping = self.get_expert_mapping()
# Streamed per-expert reload is intentionally unsupported for Quark.
for name, loaded_weight in weights:
if is_pp_missing_parameter(name, self):
continue
@@ -1018,6 +1159,9 @@ class GptOssModel(nn.Module, EagleModelMixin):
if is_pp_missing_parameter(name, self):
continue
if self._try_load_streamed_expert(name, weight, params_dict, loaded_params):
continue
if ".w13_weight" in name:
# Handle MLP gate and up projection weights
# Extract gate and up projection parts
@@ -1096,6 +1240,59 @@ class GptOssModel(nn.Module, EagleModelMixin):
loaded_params.add(name)
return loaded_params
@staticmethod
def _get_streamed_expert_info(
name: str,
params_dict: dict[str, torch.nn.Parameter],
) -> tuple[int, str, str] | None:
"""Parse ``...experts.<expert_id>.<fused_param>`` checkpoint keys."""
if ".mlp.experts." not in name:
return None
suffix = name.rsplit(".", 1)[-1]
shard_id = _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD.get(suffix)
if shard_id is None:
return None
prefix, expert_suffix = name.split(".mlp.experts.", maxsplit=1)
expert_id_str, separator, param_suffix = expert_suffix.partition(".")
if not separator or not expert_id_str.isdigit():
return None
expert_id = int(expert_id_str)
for base_layer_prefix in ("", "base_layer."):
fused_name = (
f"{prefix}.mlp.experts.{base_layer_prefix}routed_experts.{param_suffix}"
)
if fused_name in params_dict:
return expert_id, fused_name, shard_id
return None
@classmethod
def _try_load_streamed_expert(
cls,
name: str,
loaded_weight: torch.Tensor,
params_dict: dict[str, torch.nn.Parameter],
loaded_params: set[str],
) -> bool:
expert_info = cls._get_streamed_expert_info(name, params_dict)
if expert_info is None:
return False
expert_id, fused_name, shard_id = expert_info
param = params_dict[fused_name]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
success = weight_loader(
param,
loaded_weight,
weight_name=fused_name,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_params.add(fused_name)
return True
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)