[Feature][Model Loader] Apply model-owned MTP shard selection to RunAI ModelStreamer (#60759)

Signed-off-by: 浩韵 <zuohaiyu.zhy@alibaba-inc.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Haiyu Zuo
2026-10-15 12:23:11 +08:00
committed by GitHub
co-authored by Kevin H. Luu
parent 833665cded
commit ff5a91a456
6 changed files with 724 additions and 13 deletions
@@ -12,9 +12,13 @@ import gc
import weakref
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.models.interfaces import _language_model_by_module
from vllm.model_executor.models.interfaces import (
SupportsMultiModal,
_language_model_by_module,
)
pytestmark = pytest.mark.cpu_test
@@ -57,3 +61,42 @@ def test_cached_entry_is_dropped_with_its_key():
gc.collect()
assert not any(isinstance(k, _MultiModalModel) for k in _language_model_by_module)
@pytest.mark.parametrize("marked", [False, True])
@pytest.mark.parametrize("text_model", ["clip", "siglip"])
def test_text_tower_without_loader_hook_is_found(text_model, marked, caplog):
from vllm.model_executor.models.clip import CLIPTextTransformer
from vllm.model_executor.models.siglip import SiglipTextTransformer
cls = CLIPTextTransformer if text_model == "clip" else SiglipTextTransformer
# Only token embeddings are needed to exercise text tower discovery.
tower = cls.__new__(cls)
nn.Module.__init__(tower)
tower.embeddings = nn.Module()
tower.embeddings.token_embedding = nn.Embedding(4, 2)
model = nn.Module()
model.text_model = tower
model._language_model_names = ["text_model"] if marked else []
caplog.clear()
language_model = SupportsMultiModal.get_language_model(model)
assert language_model is tower
input_ids = torch.tensor([0, 1])
torch.testing.assert_close(
language_model.embed_input_ids(input_ids),
tower.embeddings.token_embedding.weight[:2],
)
assert not caplog.records
@pytest.mark.parametrize("marked", [False, True])
@pytest.mark.parametrize("method", ["embed_input_ids", "forward"])
def test_language_model_discovery_rejects_noncallable_methods(method, marked):
model = _MultiModalModel()
model._language_model_names = ["language_model"] if marked else []
setattr(model.language_model, method, None)
with pytest.raises(NotImplementedError, match="No language model found"):
SupportsMultiModal.get_language_model(model)