[Performance][Model Loader] Skip non-local expert weights during EP model loading (#37136)

Signed-off-by: esmeetu <jasonailu87@gmail.com>
This commit is contained in:
Roy Wang
2026-03-16 01:33:36 -07:00
committed by GitHub
parent a2956a0f8e
commit 821eb80c0d
4 changed files with 513 additions and 2 deletions
@@ -35,6 +35,9 @@ from vllm.model_executor.layers.quantization import (
QuantizationConfig,
get_quantization_config,
)
from vllm.model_executor.model_loader.ep_weight_filter import (
should_skip_weight,
)
from vllm.platforms import current_platform
from vllm.tracing import instrument
from vllm.utils.import_utils import PlaceholderModule
@@ -721,8 +724,14 @@ def safetensors_weights_iterator(
hf_weights_files: list[str],
use_tqdm_on_load: bool,
safetensors_load_strategy: str = "lazy",
local_expert_ids: set[int] | None = None,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files."""
"""Iterate over the weights in the model safetensor files.
When *local_expert_ids* is provided, expert weights not belonging to
this rank are skipped **before** reading from disk, which drastically
reduces storage I/O for MoE models under EP.
"""
loading_desc = "Loading safetensors checkpoint shards"
if safetensors_load_strategy == "eager":
loading_desc += " (eager)"
@@ -737,7 +746,9 @@ def safetensors_weights_iterator(
if safetensors_load_strategy == "eager":
with open(st_file, "rb") as f:
state_dict = load(f.read())
yield from state_dict.items()
for name, param in state_dict.items():
if not should_skip_weight(name, local_expert_ids):
yield name, param
elif safetensors_load_strategy == "torchao":
# we can't load flattened torchao tensor subclasses directly into the model
# instead we reconstruct the subclasses here before returning
@@ -753,6 +764,8 @@ def safetensors_weights_iterator(
with safe_open(st_file, framework="pt") as f:
state_dict = {}
for name in f.keys(): # noqa: SIM118
if should_skip_weight(name, local_expert_ids):
continue
state_dict[name] = f.get_tensor(name)
# update with leftover tensor data from previous iteration, if any
@@ -769,6 +782,8 @@ def safetensors_weights_iterator(
else:
with safe_open(st_file, framework="pt") as f:
for name in f.keys(): # noqa: SIM118
if should_skip_weight(name, local_expert_ids):
continue
param = f.get_tensor(name)
yield name, param