[Feature][Model Loader] Apply model-owned MTP shard selection to RunAI ModelStreamer (#60759)
Signed-off-by: 浩韵 <zuohaiyu.zhy@alibaba-inc.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Kevin H. Luu
parent
833665cded
commit
ff5a91a456
@@ -12,9 +12,13 @@ import gc
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.models.interfaces import _language_model_by_module
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsMultiModal,
|
||||
_language_model_by_module,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -57,3 +61,42 @@ def test_cached_entry_is_dropped_with_its_key():
|
||||
gc.collect()
|
||||
|
||||
assert not any(isinstance(k, _MultiModalModel) for k in _language_model_by_module)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marked", [False, True])
|
||||
@pytest.mark.parametrize("text_model", ["clip", "siglip"])
|
||||
def test_text_tower_without_loader_hook_is_found(text_model, marked, caplog):
|
||||
from vllm.model_executor.models.clip import CLIPTextTransformer
|
||||
from vllm.model_executor.models.siglip import SiglipTextTransformer
|
||||
|
||||
cls = CLIPTextTransformer if text_model == "clip" else SiglipTextTransformer
|
||||
# Only token embeddings are needed to exercise text tower discovery.
|
||||
tower = cls.__new__(cls)
|
||||
nn.Module.__init__(tower)
|
||||
tower.embeddings = nn.Module()
|
||||
tower.embeddings.token_embedding = nn.Embedding(4, 2)
|
||||
model = nn.Module()
|
||||
model.text_model = tower
|
||||
model._language_model_names = ["text_model"] if marked else []
|
||||
|
||||
caplog.clear()
|
||||
language_model = SupportsMultiModal.get_language_model(model)
|
||||
|
||||
assert language_model is tower
|
||||
input_ids = torch.tensor([0, 1])
|
||||
torch.testing.assert_close(
|
||||
language_model.embed_input_ids(input_ids),
|
||||
tower.embeddings.token_embedding.weight[:2],
|
||||
)
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marked", [False, True])
|
||||
@pytest.mark.parametrize("method", ["embed_input_ids", "forward"])
|
||||
def test_language_model_discovery_rejects_noncallable_methods(method, marked):
|
||||
model = _MultiModalModel()
|
||||
model._language_model_names = ["language_model"] if marked else []
|
||||
setattr(model.language_model, method, None)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="No language model found"):
|
||||
SupportsMultiModal.get_language_model(model)
|
||||
|
||||
Reference in New Issue
Block a user