From b26039b09fc97aa00f095a99eda503b7dad594ec Mon Sep 17 00:00:00 2001 From: Wyett Date: Sun, 23 Aug 2026 02:59:08 -0700 Subject: [PATCH] Add routed expert loading for gpt-oss (#52209) Signed-off-by: Wyett Signed-off-by: wyett --- .../test_gpt_oss_weight_loading.py | 152 ++++++++++++++ .../model_loader/test_reload.py | 15 ++ .../model_loader/reload/meta.py | 9 +- vllm/model_executor/models/gpt_oss.py | 197 ++++++++++++++++++ 4 files changed, 370 insertions(+), 3 deletions(-) create mode 100644 tests/model_executor/model_loader/test_gpt_oss_weight_loading.py diff --git a/tests/model_executor/model_loader/test_gpt_oss_weight_loading.py b/tests/model_executor/model_loader/test_gpt_oss_weight_loading.py new file mode 100644 index 0000000000..592e65a81b --- /dev/null +++ b/tests/model_executor/model_loader/test_gpt_oss_weight_loading.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch + +from vllm.model_executor.models.gpt_oss import ( + GptOssForCausalLM, + GptOssModel, + GptOssRoutedExperts, +) + + +class _TestExperts(GptOssRoutedExperts): + def __init__( + self, + param_name: str, + shape: tuple[int, ...], + *, + tp_rank: int = 0, + quant_method_name: str = "TestQuantMethod", + weight_dtype: str = "", + dtype: torch.dtype = torch.float32, + ) -> None: + torch.nn.Module.__init__(self) + self.moe_config = SimpleNamespace( + moe_parallel_config=SimpleNamespace(tp_size=2, tp_rank=tp_rank) + ) + self.quant_method = type( + quant_method_name, (), {"weight_dtype": weight_dtype} + )() + + param = torch.nn.Parameter(torch.zeros(shape, dtype=dtype), requires_grad=False) + param.weight_loader = self.weight_loader + self.register_parameter(param_name, param) + + @staticmethod + def _map_global_expert_id_to_local_expert_id(expert_id: int) -> int: + return {3: 0, 7: 1}.get(expert_id, -1) + + +def _load( + experts: _TestExperts, + param_name: str, + loaded_weight: torch.Tensor, + shard_id: str, + expert_id: int = 3, +) -> bool: + param = getattr(experts, param_name) + return param.weight_loader( + param, + loaded_weight, + weight_name=f"model.layers.0.mlp.experts.routed_experts.{param_name}", + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ) + + +@pytest.mark.parametrize("base_layer_prefix", ["", "base_layer."]) +def test_streamed_expert_resolves_wrapped_parameter(base_layer_prefix: str) -> None: + experts = _TestExperts("w13_weight", (2, 4, 3)) + checkpoint_name = "model.layers.2.mlp.experts.7.gate_up_proj" + mapped_name = GptOssForCausalLM.hf_to_vllm_mapper.apply_list([checkpoint_name])[0] + fused_name = ( + f"model.layers.2.mlp.experts.{base_layer_prefix}routed_experts.w13_weight" + ) + params_dict = {fused_name: experts.w13_weight} + loaded_params: set[str] = set() + loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8) + + assert GptOssModel._get_streamed_expert_info(mapped_name, params_dict) == ( + 7, + fused_name, + "gpt_oss_w13", + ) + assert GptOssModel._try_load_streamed_expert( + mapped_name, loaded_weight, params_dict, loaded_params + ) + assert loaded_params == {fused_name} + assert torch.equal(experts.w13_weight[1], loaded_weight[:, :4].t()) + assert torch.count_nonzero(experts.w13_weight[0]) == 0 + + +def test_streamed_expert_skips_non_local_expert() -> None: + experts = _TestExperts("w2_bias", (2, 3)) + + assert not _load(experts, "w2_bias", torch.ones(3), "gpt_oss_w2", expert_id=6) + assert torch.count_nonzero(experts.w2_bias) == 0 + + +@pytest.mark.parametrize("param_name", ["w13_weight", "w2_weight"]) +def test_streamed_unquantized_expert_tp_slice(param_name: str) -> None: + if param_name == "w13_weight": + experts = _TestExperts(param_name, (2, 4, 3), tp_rank=1) + loaded_weight = torch.arange(24, dtype=torch.float32).reshape(3, 8) + shard_id = "gpt_oss_w13" + expected = loaded_weight[:, 4:].t() + else: + experts = _TestExperts(param_name, (2, 3, 2), tp_rank=1) + loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3) + shard_id = "gpt_oss_w2" + expected = loaded_weight[2:].t() + + assert _load(experts, param_name, loaded_weight, shard_id) + assert torch.equal(getattr(experts, param_name)[0], expected) + + +@pytest.mark.parametrize("tp_rank", [0, 1]) +def test_streamed_down_bias_owned_by_tp_rank_zero(tp_rank: int) -> None: + experts = _TestExperts("w2_bias", (2, 3), tp_rank=tp_rank) + loaded_weight = torch.arange(1, 4, dtype=torch.float32) + + assert _load(experts, "w2_bias", loaded_weight, "gpt_oss_w2") + expected = loaded_weight if tp_rank == 0 else torch.zeros_like(loaded_weight) + assert torch.equal(experts.w2_bias[0], expected) + + +def test_streamed_gate_up_bias_tp_slice() -> None: + experts = _TestExperts("w13_bias", (2, 4), tp_rank=1) + loaded_weight = torch.arange(8, dtype=torch.float32) + + assert _load(experts, "w13_bias", loaded_weight, "gpt_oss_w13") + assert torch.equal(experts.w13_bias[0], loaded_weight[4:]) + + +@pytest.mark.parametrize("case", ["mxfp4", "nvfp4"]) +def test_streamed_quantized_expert_layout(case: str) -> None: + if case == "mxfp4": + experts = _TestExperts( + "w13_weight", + (2, 4, 4), + tp_rank=1, + weight_dtype="gpt_oss_mxfp4", + dtype=torch.uint8, + ) + loaded_weight = torch.arange(32, dtype=torch.uint8).reshape(8, 2, 2) + expected = loaded_weight.reshape(8, 4)[4:] + else: + experts = _TestExperts( + "w13_weight", + (2, 4, 2), + tp_rank=1, + quant_method_name="Nvfp4OnlineMoEMethod", + dtype=torch.uint8, + ) + loaded_weight = torch.arange(16, dtype=torch.uint8).reshape(8, 2) + expected = loaded_weight[4:] + + assert _load(experts, "w13_weight", loaded_weight, "gpt_oss_w13") + assert torch.equal(experts.w13_weight[0], expected) diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index f32ccb118f..b17f2ece3c 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -218,6 +218,21 @@ def test_reload_lifecycle(): assert tensor.__dict__ == materialized_tensor.__dict__ +def test_restore_layer_replaces_postprocessed_tensor_attribute(): + layer = torch.nn.Linear(2, 3, bias=False) + info = LayerReloadingInfo( + restore_metadata=capture_layer_to_meta(layer), + restore_device=torch.device("cpu"), + ) + del layer.weight + layer.weight = torch.empty(3, 2) + + restore_layer_on_meta(layer, info) + + assert isinstance(layer.weight, torch.nn.Parameter) + assert layer.weight.is_meta + + def test_materialize_layer_preserves_non_meta_tensors(): """Ensure that materialize_layer does not overwrite non meta tensors.""" layer = torch.nn.Linear(2, 3, bias=True) diff --git a/vllm/model_executor/model_loader/reload/meta.py b/vllm/model_executor/model_loader/reload/meta.py index 213a9aaeaf..a8f8023cf2 100644 --- a/vllm/model_executor/model_loader/reload/meta.py +++ b/vllm/model_executor/model_loader/reload/meta.py @@ -124,11 +124,14 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo): return non_persistent = set(layer._non_persistent_buffers_set) - for name in get_layer_tensors(layer): - if name not in SKIP_TENSORS: + restore_params, restore_buffers = info.restore_metadata + tensor_names = ( + get_layer_tensors(layer).keys() | restore_params.keys() | restore_buffers.keys() + ) + for name in tensor_names: + if name not in SKIP_TENSORS and hasattr(layer, name): delattr(layer, name) - restore_params, restore_buffers = info.restore_metadata for name, param in restore_params.items(): if name not in SKIP_TENSORS: param = restore_layer_refs(param, layer) diff --git a/vllm/model_executor/models/gpt_oss.py b/vllm/model_executor/models/gpt_oss.py index d5fbee384a..7f4de57f41 100644 --- a/vllm/model_executor/models/gpt_oss.py +++ b/vllm/model_executor/models/gpt_oss.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig +from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -68,6 +69,15 @@ from .utils import ( maybe_prefix, ) +_GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = { + "w13_weight": "gpt_oss_w13", + "w2_weight": "gpt_oss_w2", + "w13_bias": "gpt_oss_w13", + "w2_bias": "gpt_oss_w2", + "w13_weight_scale": "gpt_oss_w13", + "w2_weight_scale": "gpt_oss_w2", +} + class OAIAttention(nn.Module): # Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves); @@ -179,6 +189,132 @@ class OAIAttention(nn.Module): return output +class GptOssRoutedExperts(RoutedExperts): + """Load one GPT-OSS expert at a time without assembling the global stack.""" + + @staticmethod + def _narrow_for_rank( + loaded_weight: torch.Tensor, + dim: int, + rank: int, + size: int, + ) -> torch.Tensor: + start = rank * size + available = loaded_weight.shape[dim] - start + return loaded_weight.narrow(dim, start, min(size, max(available, 0))) + + @staticmethod + def _copy_to_expert(expert_data: torch.Tensor, loaded_weight: torch.Tensor) -> None: + if expert_data.numel() == loaded_weight.numel() == 1: + expert_data.copy_(loaded_weight.reshape(())) + return + while loaded_weight.ndim > expert_data.ndim and loaded_weight.shape[-1] == 1: + loaded_weight = loaded_weight.squeeze(-1) + slices = tuple(slice(0, size) for size in loaded_weight.shape) + expert_data[slices].copy_(loaded_weight) + + def _load_expert_bias( + self, + expert_data: torch.Tensor, + loaded_weight: torch.Tensor, + shard_id: str, + ) -> None: + tp_rank = self.moe_config.moe_parallel_config.tp_rank + if shard_id == "gpt_oss_w13": + loaded_weight = self._narrow_for_rank( + loaded_weight, 0, tp_rank, expert_data.shape[0] + ) + elif tp_rank != 0: + loaded_weight = torch.zeros_like(loaded_weight) + self._copy_to_expert(expert_data, loaded_weight) + + def _load_unquantized_expert( + self, + expert_data: torch.Tensor, + loaded_weight: torch.Tensor, + shard_id: str, + ) -> None: + tp_rank = self.moe_config.moe_parallel_config.tp_rank + if shard_id == "gpt_oss_w13": + loaded_weight = self._narrow_for_rank( + loaded_weight, 1, tp_rank, expert_data.shape[0] + ) + loaded_weight = loaded_weight.t().contiguous() + else: + loaded_weight = self._narrow_for_rank( + loaded_weight, 0, tp_rank, expert_data.shape[1] + ) + loaded_weight = loaded_weight.t().contiguous() + self._copy_to_expert(expert_data, loaded_weight) + + def _load_packed_expert( + self, + expert_data: torch.Tensor, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + ) -> None: + tp_rank = self.moe_config.moe_parallel_config.tp_rank + is_w13 = shard_id == "gpt_oss_w13" + is_weight = weight_name.endswith("_weight") + is_partitioned_scale = weight_name.endswith("_weight_scale") + + if is_weight: + loaded_weight = loaded_weight.reshape(loaded_weight.shape[0], -1) + + if is_w13 and (is_weight or is_partitioned_scale): + loaded_weight = self._narrow_for_rank( + loaded_weight, 0, tp_rank, expert_data.shape[0] + ) + elif not is_w13 and (is_weight or is_partitioned_scale): + loaded_weight = self._narrow_for_rank( + loaded_weight, -1, tp_rank, expert_data.shape[-1] + ) + + self._copy_to_expert(expert_data, loaded_weight) + + def weight_loader( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: bool = False, + ) -> bool | None: + if shard_id not in ("gpt_oss_w13", "gpt_oss_w2"): + return super().weight_loader( + param, + loaded_weight, + weight_name, + shard_id, + expert_id, + return_success, + ) + + expert_id = self._map_global_expert_id_to_local_expert_id(expert_id) + if expert_id == -1: + return False if return_success else None + + expert_data = param.data[expert_id] + weight_dtype = getattr(self.quant_method, "weight_dtype", "") + quant_method_name = self.quant_method.__class__.__name__ + + if weight_name.endswith("_bias"): + self._load_expert_bias(expert_data, loaded_weight, shard_id) + elif weight_dtype in ( + "gpt_oss_mxfp4", + "mxfp4", + ) or quant_method_name in ( + "CompressedTensorsW4A4Nvfp4MoEMethod", + "Nvfp4OnlineMoEMethod", + ): + self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id) + else: + self._load_unquantized_expert(expert_data, loaded_weight, shard_id) + return True if return_success else None + + class MLPBlock(torch.nn.Module): def __init__( self, @@ -223,6 +359,7 @@ class MLPBlock(torch.nn.Module): has_bias=True, activation="swigluoai", is_sequence_parallel=self.is_sequence_parallel, + routed_experts_cls=GptOssRoutedExperts, ) def forward(self, x: torch.Tensor) -> torch.Tensor: @@ -416,6 +553,9 @@ class GptOssModel(nn.Module, EagleModelMixin): if is_pp_missing_parameter(name, self): continue + if self._try_load_streamed_expert(name, weight, params_dict, loaded_params): + continue + if ".w13_weight_scale" in name: # Handle MLP gate and up projection weights scale if use_ep: @@ -634,6 +774,7 @@ class GptOssModel(nn.Module, EagleModelMixin): tp_rank_start = tp_rank * per_rank_intermediate_size tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size) expert_params_mapping = self.get_expert_mapping() + # Streamed per-expert reload is intentionally unsupported for Quark. for name, loaded_weight in weights: if is_pp_missing_parameter(name, self): continue @@ -1018,6 +1159,9 @@ class GptOssModel(nn.Module, EagleModelMixin): if is_pp_missing_parameter(name, self): continue + if self._try_load_streamed_expert(name, weight, params_dict, loaded_params): + continue + if ".w13_weight" in name: # Handle MLP gate and up projection weights # Extract gate and up projection parts @@ -1096,6 +1240,59 @@ class GptOssModel(nn.Module, EagleModelMixin): loaded_params.add(name) return loaded_params + @staticmethod + def _get_streamed_expert_info( + name: str, + params_dict: dict[str, torch.nn.Parameter], + ) -> tuple[int, str, str] | None: + """Parse ``...experts..`` checkpoint keys.""" + if ".mlp.experts." not in name: + return None + suffix = name.rsplit(".", 1)[-1] + shard_id = _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD.get(suffix) + if shard_id is None: + return None + + prefix, expert_suffix = name.split(".mlp.experts.", maxsplit=1) + expert_id_str, separator, param_suffix = expert_suffix.partition(".") + if not separator or not expert_id_str.isdigit(): + return None + expert_id = int(expert_id_str) + for base_layer_prefix in ("", "base_layer."): + fused_name = ( + f"{prefix}.mlp.experts.{base_layer_prefix}routed_experts.{param_suffix}" + ) + if fused_name in params_dict: + return expert_id, fused_name, shard_id + return None + + @classmethod + def _try_load_streamed_expert( + cls, + name: str, + loaded_weight: torch.Tensor, + params_dict: dict[str, torch.nn.Parameter], + loaded_params: set[str], + ) -> bool: + expert_info = cls._get_streamed_expert_info(name, params_dict) + if expert_info is None: + return False + + expert_id, fused_name, shard_id = expert_info + param = params_dict[fused_name] + weight_loader = typing.cast(Callable[..., bool], param.weight_loader) + success = weight_loader( + param, + loaded_weight, + weight_name=fused_name, + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ) + if success: + loaded_params.add(fused_name) + return True + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ # (param_name, shard_name, shard_id)