[Feature][Model Loader] Apply model-owned MTP shard selection to RunAI ModelStreamer (#60759)

Signed-off-by: 浩韵 <zuohaiyu.zhy@alibaba-inc.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Haiyu Zuo
2026-10-15 12:23:11 +08:00
committed by GitHub
co-authored by Kevin H. Luu
parent 833665cded
commit ff5a91a456
6 changed files with 724 additions and 13 deletions
@@ -1,7 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
from collections.abc import Generator
import posixpath
import tempfile
from collections.abc import Callable, Generator
from typing import Any
from urllib.parse import urlsplit
import torch
from torch import nn
@@ -9,14 +14,110 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.distributed import get_pp_group, get_world_group
from vllm.logger import init_logger
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.weight_utils import (
download_safetensors_index_file_from_hf,
download_weights_from_hf,
filter_safetensors_files_by_weight_name,
runai_safetensors_weights_iterator,
)
from vllm.transformers_utils.repo_utils import resolve_revision
from vllm.transformers_utils.runai_utils import is_runai_obj_uri, list_safetensors
from vllm.transformers_utils.runai_utils import (
is_runai_obj_uri,
list_safetensors,
runai_pull_files,
)
logger = init_logger(__name__)
def _get_remote_checkpoint_keys(files: list[str]) -> list[list[str]]:
from runai_model_streamer import DistributedStreamer
from runai_model_streamer.safetensors_streamer.safetensors_pytorch import (
prepare_request,
)
# RunAI's metadata reader uses header ranges and the streamer's backend.
with DistributedStreamer() as streamer:
metadata = prepare_request(streamer, files, None)
return [[tensor.name for tensor in tensors] for _, tensors, _ in metadata]
def _filter_remote_weights(
files: list[str],
source: str,
is_unused_weight: Callable[[str], bool],
) -> list[str]:
try:
with tempfile.TemporaryDirectory() as directory:
# Backends may match the full object key, not its basename.
index_key = posixpath.join(
urlsplit(source).path.strip("/"), SAFE_WEIGHTS_INDEX_NAME
)
runai_pull_files(
source.rstrip("/") + "/",
directory,
allow_pattern=[index_key, SAFE_WEIGHTS_INDEX_NAME],
)
with open(os.path.join(directory, SAFE_WEIGHTS_INDEX_NAME)) as index:
weight_map = json.load(index)["weight_map"]
if not isinstance(weight_map, dict) or not weight_map:
raise ValueError("Empty or invalid weight map")
if any(
not isinstance(name, str)
or not isinstance(shard, str)
or not shard
or posixpath.isabs(shard)
or posixpath.normpath(shard) != shard
or shard in (".", "..")
or shard.startswith("../")
for name, shard in weight_map.items()
):
raise ValueError("Invalid checkpoint index entry")
except Exception:
# Optional metadata must not make an otherwise loadable model fail.
# Model-rule and tensor-read errors remain outside this boundary.
logger.warning(
"Cannot use checkpoint index for RunAI shard selection; "
"keeping the original file list"
)
return files
# Unindexed files cannot be excluded using this metadata.
root = source.rstrip("/") + "/"
indexed = {root + shard for shard in weight_map.values()}
needed = {
root + shard for name, shard in weight_map.items() if not is_unused_weight(name)
}
if not needed.issubset(files):
logger.warning(
"Checkpoint index references missing required RunAI shards; "
"keeping the original file list"
)
return files
kept = [path for path in files if path not in indexed or path in needed]
excluded = [path for path in files if path not in kept]
if excluded and kept:
try:
headers = _get_remote_checkpoint_keys(excluded)
if len(headers) != len(excluded):
raise ValueError("Incomplete checkpoint headers")
except Exception:
logger.warning(
"Cannot verify excluded RunAI shard headers; "
"keeping the original file list"
)
return files
# A stale index can omit wanted tensors inside an indexed shard.
if any(not is_unused_weight(name) for names in headers for name in names):
logger.warning(
"Checkpoint index disagrees with RunAI shard headers; "
"keeping the original file list"
)
return files
return kept or files
class RunaiModelStreamerLoader(BaseModelLoader):
@@ -122,12 +223,43 @@ class RunaiModelStreamerLoader(BaseModelLoader):
return hf_weights_files
def _get_weights_iterator(
self, model_or_path: str, revision: str | None
self,
model_or_path: str,
revision: str | None,
is_unused_weight: Callable[[str], bool] | None = None,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Get an iterator for the model weights based on the load format."""
hf_weights_files = self._prepare_weights(model_or_path, revision)
selected_files = hf_weights_files
distributed = self._is_distributed and torch.distributed.is_initialized()
# Drafts load only on the last PP stage; RunAI is invoked without a
# model-specific group, so retain its existing full-list path with PP.
can_select = not distributed or get_pp_group().world_size == 1
if is_unused_weight is not None and can_select:
if is_runai_obj_uri(model_or_path):
selected_files = _filter_remote_weights(
hf_weights_files, model_or_path, is_unused_weight
)
else:
selected_files = filter_safetensors_files_by_weight_name(
hf_weights_files, is_unused_weight
)
if distributed and can_select:
# RunAI broadcasts across node-local or WORLD groups, not just TP.
# A rank that cannot filter safely keeps the group on the full list.
group = get_world_group()
selections: list[Any] = [None] * group.world_size
torch.distributed.all_gather_object(
selections, (hf_weights_files, selected_files), group=group.cpu_group
)
if all(files == hf_weights_files for files, _ in selections):
needed = {path for _, selected in selections for path in selected}
selected_files = [path for path in hf_weights_files if path in needed]
else:
# Do not substitute another node's local checkpoint paths.
selected_files = hf_weights_files
return runai_safetensors_weights_iterator(
hf_weights_files, self.load_config.use_tqdm_on_load, self._is_distributed
selected_files, self.load_config.use_tqdm_on_load, self._is_distributed
)
def download_model(self, model_config: ModelConfig) -> None:
@@ -142,7 +274,11 @@ class RunaiModelStreamerLoader(BaseModelLoader):
model_weights = model_config.model
if model_weights_override := model_config.model_weights:
model_weights = model_weights_override
yield from self._get_weights_iterator(model_weights, model_config.revision)
yield from self._get_weights_iterator(
model_weights,
model_config.revision,
getattr(model, "is_unused_checkpoint_weight", None),
)
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
"""Load weights into a model."""
+9 -2
View File
@@ -141,6 +141,13 @@ _language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = (
)
def _is_language_model(model: object) -> TypeIs[VllmModel]:
# Text towers can use config/position_ids signatures and omit loader hooks.
return callable(getattr(model, "embed_input_ids", None)) and callable(
getattr(model, "forward", None)
)
@runtime_checkable
class SupportsMultiModalEmbeddings(Protocol):
"""The interface for models that can merge external multimodal embeddings."""
@@ -284,14 +291,14 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
mod = getattr(mod, attr)
candidate: object = mod
if candidate is not self_module and isinstance(candidate, VllmModel):
if candidate is not self_module and _is_language_model(candidate):
_language_model_by_module[self_module] = candidate
return candidate
# Fallback
for mod in self_module.children():
candidate = mod
if isinstance(candidate, VllmModel):
if _is_language_model(candidate):
_language_model_by_module[self_module] = candidate
return candidate
+11 -4
View File
@@ -51,6 +51,15 @@ class VllmModel(Protocol):
self, input_ids: torch.Tensor, positions: torch.Tensor
) -> torch.Tensor: ...
def is_unused_checkpoint_weight(self, name: str) -> bool:
"""Return True for checkpoint weights this model does not load.
Defining this method is optional for satisfying this interface.
Override with the same rule used in ``load_weights``. Models without
a selection rule keep every checkpoint weight by default.
"""
return False
def _check_vllm_model_init(model: type[object] | object) -> bool:
model_init = getattr(model, "__init__") # noqa: B009
@@ -134,10 +143,8 @@ def is_text_generation_model(
if not is_vllm_model(model):
return False
if isinstance(model, type):
return isinstance(model, VllmModelForTextGeneration)
return isinstance(model, VllmModelForTextGeneration)
# Optional protocol defaults are not inherited by structural models.
return callable(getattr(model, "compute_logits", None))
@runtime_checkable