[Performance][Model Loader] Skip non-local expert weights during EP model loading (#37136)

Signed-off-by: esmeetu <jasonailu87@gmail.com>
This commit is contained in:
Roy Wang
2026-03-16 01:33:36 -07:00
committed by GitHub
parent a2956a0f8e
commit 821eb80c0d
4 changed files with 513 additions and 2 deletions
@@ -16,6 +16,9 @@ from vllm.config.load import LoadConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.ep_weight_filter import (
compute_local_expert_ids,
)
from vllm.model_executor.model_loader.weight_utils import (
download_safetensors_index_file_from_hf,
download_weights_from_hf,
@@ -70,6 +73,7 @@ class DefaultModelLoader(BaseModelLoader):
def __init__(self, load_config: LoadConfig):
super().__init__(load_config)
self.local_expert_ids: set[int] | None = None
extra_config = load_config.model_loader_extra_config
allowed_keys = {"enable_multithread_load", "num_threads"}
@@ -243,6 +247,7 @@ class DefaultModelLoader(BaseModelLoader):
hf_weights_files,
self.load_config.use_tqdm_on_load,
self.load_config.safetensors_load_strategy,
local_expert_ids=self.local_expert_ids,
)
else:
if extra_config.get("enable_multithread_load"):
@@ -296,6 +301,58 @@ class DefaultModelLoader(BaseModelLoader):
allow_patterns_overrides=None,
)
def _init_ep_weight_filter(self, model_config: ModelConfig) -> None:
"""Compute local expert ids for EP weight filtering.
When expert parallelism is active, each rank only needs a subset of
expert weights. By computing the set upfront we can skip non-local
expert tensors *before* reading them from disk.
"""
from vllm.config import get_current_vllm_config
vllm_config = get_current_vllm_config()
parallel_config = vllm_config.parallel_config
if not (model_config.is_moe and parallel_config.enable_expert_parallel):
return
num_experts = model_config.get_num_experts()
if num_experts <= 0:
return
# EP size/rank computation mirrors FusedMoEParallelConfig.make():
# ep_size = dp_size * pcp_size * tp_size (flattened)
# ep_rank = dp_rank * pcp_size * tp_size + pcp_rank * tp_size + tp_rank
from vllm.distributed import (
get_dp_group,
get_pcp_group,
get_tensor_model_parallel_rank,
)
dp_size = parallel_config.data_parallel_size
tp_size = parallel_config.tensor_parallel_size
pcp_size = parallel_config.prefill_context_parallel_size
dp_rank = get_dp_group().rank_in_group if dp_size > 1 else 0
tp_rank = get_tensor_model_parallel_rank() if tp_size > 1 else 0
pcp_rank = get_pcp_group().rank_in_group if pcp_size > 1 else 0
ep_size = dp_size * pcp_size * tp_size
ep_rank = dp_rank * pcp_size * tp_size + pcp_rank * tp_size + tp_rank
self.local_expert_ids = compute_local_expert_ids(
num_experts,
ep_size,
ep_rank,
placement=parallel_config.expert_placement_strategy,
)
if self.local_expert_ids is not None:
logger.info_once(
"EP weight filter: ep_size=%d, ep_rank=%d, loading %d/%d experts",
ep_size,
ep_rank,
len(self.local_expert_ids),
num_experts,
)
@instrument(span_name="Load weights")
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
if model_config.quantization == "torchao":
@@ -307,6 +364,8 @@ class DefaultModelLoader(BaseModelLoader):
):
self.load_config.safetensors_load_strategy = "torchao"
self._init_ep_weight_filter(model_config)
weights_to_load = {name for name, _ in model.named_parameters()}
loaded_weights = model.load_weights(self.get_all_weights(model_config, model))