[Model] Explicit interface for vLLM models and support OOT embedding models (#9108)
This commit is contained in:
@@ -1,10 +1,17 @@
|
||||
from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal,
|
||||
SupportsPP, has_inner_state, supports_lora,
|
||||
supports_multimodal, supports_pp)
|
||||
from .interfaces_base import (VllmModelForEmbedding,
|
||||
VllmModelForTextGeneration, is_embedding_model,
|
||||
is_text_generation_model)
|
||||
from .registry import ModelRegistry
|
||||
|
||||
__all__ = [
|
||||
"ModelRegistry",
|
||||
"VllmModelForEmbedding",
|
||||
"is_embedding_model",
|
||||
"VllmModelForTextGeneration",
|
||||
"is_text_generation_model",
|
||||
"HasInnerState",
|
||||
"has_inner_state",
|
||||
"SupportsLoRA",
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import inspect
|
||||
from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
|
||||
Protocol, Type, Union, overload, runtime_checkable)
|
||||
|
||||
@@ -6,9 +5,9 @@ import torch
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import supports_kw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.attention import AttentionMetadata
|
||||
from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -142,9 +141,7 @@ def supports_lora(
|
||||
return result
|
||||
|
||||
|
||||
def _supports_lora(
|
||||
model: Union[Type[object], object],
|
||||
) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
|
||||
def _supports_lora(model: Union[Type[object], object]) -> bool:
|
||||
if isinstance(model, type):
|
||||
return isinstance(model, _SupportsLoRAType)
|
||||
|
||||
@@ -175,10 +172,7 @@ class SupportsPP(Protocol):
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
kv_caches: List[torch.Tensor],
|
||||
attn_metadata: "AttentionMetadata",
|
||||
*,
|
||||
intermediate_tensors: Optional["IntermediateTensors"],
|
||||
) -> Union[torch.Tensor, "IntermediateTensors"]:
|
||||
"""
|
||||
@@ -205,10 +199,7 @@ class _SupportsPPType(Protocol):
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
kv_caches: List[torch.Tensor],
|
||||
attn_metadata: "AttentionMetadata",
|
||||
*,
|
||||
intermediate_tensors: Optional["IntermediateTensors"],
|
||||
) -> Union[torch.Tensor, "IntermediateTensors"]:
|
||||
...
|
||||
@@ -257,24 +248,19 @@ def supports_pp(
|
||||
return supports_attributes and supports_inspect
|
||||
|
||||
|
||||
def _supports_pp_attributes(
|
||||
model: Union[Type[object], object],
|
||||
) -> Union[bool, TypeIs[Type[SupportsPP]], TypeIs[SupportsPP]]:
|
||||
def _supports_pp_attributes(model: Union[Type[object], object]) -> bool:
|
||||
if isinstance(model, type):
|
||||
return isinstance(model, _SupportsPPType)
|
||||
|
||||
return isinstance(model, SupportsPP)
|
||||
|
||||
|
||||
def _supports_pp_inspect(
|
||||
model: Union[Type[object], object],
|
||||
) -> Union[bool, TypeIs[Type[SupportsPP]], TypeIs[SupportsPP]]:
|
||||
def _supports_pp_inspect(model: Union[Type[object], object]) -> bool:
|
||||
model_forward = getattr(model, "forward", None)
|
||||
if not callable(model_forward):
|
||||
return False
|
||||
|
||||
forward_params = inspect.signature(model_forward).parameters
|
||||
return "intermediate_tensors" in forward_params
|
||||
return supports_kw(model_forward, "intermediate_tensors")
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
from typing import (TYPE_CHECKING, List, Optional, Protocol, Type, Union,
|
||||
overload, runtime_checkable)
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
from typing_extensions import TypeIs, TypeVar
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import supports_kw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.attention import AttentionMetadata
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.model_executor.layers.pooler import PoolerOutput
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# The type of HF config
|
||||
C_co = TypeVar("C_co", bound=PretrainedConfig, covariant=True)
|
||||
|
||||
# The type of hidden states
|
||||
# Currently, T = torch.Tensor for all models except for Medusa
|
||||
# which has T = List[torch.Tensor]
|
||||
T = TypeVar("T", default=torch.Tensor)
|
||||
T_co = TypeVar("T_co", default=torch.Tensor, covariant=True)
|
||||
|
||||
# NOTE: Unlike those in `interfaces.py`, we don't define `ClassVar` tags
|
||||
# for the base interfaces to avoid breaking OOT registration for existing models
|
||||
# that don't inherit from the base interface classes
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VllmModel(Protocol[C_co, T_co]):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: C_co,
|
||||
*,
|
||||
cache_config: Optional["CacheConfig"],
|
||||
quant_config: Optional["QuantizationConfig"],
|
||||
) -> None:
|
||||
...
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
kv_caches: List[torch.Tensor],
|
||||
attn_metadata: "AttentionMetadata",
|
||||
) -> T_co:
|
||||
...
|
||||
|
||||
|
||||
def _check_vllm_model_init(model: Union[Type[object], object]) -> bool:
|
||||
model_init = model.__init__
|
||||
vllm_kws = ("cache_config", "quant_config")
|
||||
missing_kws = tuple(kw for kw in vllm_kws
|
||||
if not supports_kw(model_init, kw))
|
||||
|
||||
if missing_kws and (isinstance(model, type)
|
||||
and issubclass(model, nn.Module)):
|
||||
logger.warning(
|
||||
"The model (%s) is missing "
|
||||
"vLLM-specific keywords from its initializer: %s",
|
||||
model,
|
||||
missing_kws,
|
||||
)
|
||||
|
||||
return len(missing_kws) == 0
|
||||
|
||||
|
||||
def _check_vllm_model_forward(model: Union[Type[object], object]) -> bool:
|
||||
model_forward = getattr(model, "forward", None)
|
||||
if not callable(model_forward):
|
||||
return False
|
||||
|
||||
vllm_kws = ("input_ids", "positions", "kv_caches", "attn_metadata")
|
||||
missing_kws = tuple(kw for kw in vllm_kws
|
||||
if not supports_kw(model_forward, kw))
|
||||
|
||||
if missing_kws and (isinstance(model, type)
|
||||
and issubclass(model, nn.Module)):
|
||||
logger.warning(
|
||||
"The model (%s) is missing "
|
||||
"vLLM-specific keywords from its initializer: %s",
|
||||
model,
|
||||
missing_kws,
|
||||
)
|
||||
|
||||
return len(missing_kws) == 0
|
||||
|
||||
|
||||
@overload
|
||||
def is_vllm_model(model: Type[object]) -> TypeIs[Type[VllmModel]]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def is_vllm_model(model: object) -> TypeIs[VllmModel]:
|
||||
...
|
||||
|
||||
|
||||
def is_vllm_model(
|
||||
model: Union[Type[object], object],
|
||||
) -> Union[TypeIs[Type[VllmModel]], TypeIs[VllmModel]]:
|
||||
return _check_vllm_model_init(model) and _check_vllm_model_forward(model)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VllmModelForTextGeneration(VllmModel[C_co, T], Protocol[C_co, T]):
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: T,
|
||||
sampling_metadata: "SamplingMetadata",
|
||||
) -> Optional[T]:
|
||||
"""Return `None` if TP rank > 0."""
|
||||
...
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: T,
|
||||
sampling_metadata: "SamplingMetadata",
|
||||
) -> "SamplerOutput":
|
||||
"""Only called on TP rank 0."""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def is_text_generation_model(
|
||||
model: Type[object]) -> TypeIs[Type[VllmModelForTextGeneration]]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def is_text_generation_model(
|
||||
model: object) -> TypeIs[VllmModelForTextGeneration]:
|
||||
...
|
||||
|
||||
|
||||
def is_text_generation_model(
|
||||
model: Union[Type[object], object],
|
||||
) -> Union[TypeIs[Type[VllmModelForTextGeneration]],
|
||||
TypeIs[VllmModelForTextGeneration]]:
|
||||
if not is_vllm_model(model):
|
||||
return False
|
||||
|
||||
if isinstance(model, type):
|
||||
return isinstance(model, VllmModelForTextGeneration)
|
||||
|
||||
return isinstance(model, VllmModelForTextGeneration)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]):
|
||||
|
||||
def pooler(
|
||||
self,
|
||||
hidden_states: T,
|
||||
pooling_metadata: "PoolingMetadata",
|
||||
) -> "PoolerOutput":
|
||||
"""Only called on TP rank 0."""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def is_embedding_model(
|
||||
model: Type[object]) -> TypeIs[Type[VllmModelForEmbedding]]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def is_embedding_model(model: object) -> TypeIs[VllmModelForEmbedding]:
|
||||
...
|
||||
|
||||
|
||||
def is_embedding_model(
|
||||
model: Union[Type[object], object],
|
||||
) -> Union[TypeIs[Type[VllmModelForEmbedding]], TypeIs[VllmModelForEmbedding]]:
|
||||
if not is_vllm_model(model):
|
||||
return False
|
||||
|
||||
if isinstance(model, type):
|
||||
return isinstance(model, VllmModelForEmbedding)
|
||||
|
||||
return isinstance(model, VllmModelForEmbedding)
|
||||
@@ -12,10 +12,12 @@ from vllm.logger import init_logger
|
||||
from vllm.utils import is_hip
|
||||
|
||||
from .interfaces import supports_multimodal, supports_pp
|
||||
from .interfaces_base import is_embedding_model, is_text_generation_model
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_GENERATION_MODELS = {
|
||||
_TEXT_GENERATION_MODELS = {
|
||||
# [Decoder-only]
|
||||
"AquilaModel": ("llama", "LlamaForCausalLM"),
|
||||
"AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
|
||||
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
|
||||
@@ -74,10 +76,9 @@ _GENERATION_MODELS = {
|
||||
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
|
||||
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
|
||||
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
|
||||
# NOTE: The below models are for speculative decoding only
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"EAGLEModel": ("eagle", "EAGLE"),
|
||||
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
|
||||
# [Encoder-decoder]
|
||||
"BartModel": ("bart", "BartForConditionalGeneration"),
|
||||
"BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
|
||||
}
|
||||
|
||||
_EMBEDDING_MODELS = {
|
||||
@@ -114,16 +115,18 @@ _MULTIMODAL_MODELS = {
|
||||
"MllamaForConditionalGeneration": ("mllama",
|
||||
"MllamaForConditionalGeneration"),
|
||||
}
|
||||
_CONDITIONAL_GENERATION_MODELS = {
|
||||
"BartModel": ("bart", "BartForConditionalGeneration"),
|
||||
"BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
|
||||
|
||||
_SPECULATIVE_DECODING_MODELS = {
|
||||
"EAGLEModel": ("eagle", "EAGLE"),
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
|
||||
}
|
||||
|
||||
_MODELS = {
|
||||
**_GENERATION_MODELS,
|
||||
**_TEXT_GENERATION_MODELS,
|
||||
**_EMBEDDING_MODELS,
|
||||
**_MULTIMODAL_MODELS,
|
||||
**_CONDITIONAL_GENERATION_MODELS,
|
||||
**_SPECULATIVE_DECODING_MODELS,
|
||||
}
|
||||
|
||||
# Architecture -> type or (module, class).
|
||||
@@ -317,6 +320,19 @@ class ModelRegistry:
|
||||
|
||||
return result.returncode == 0
|
||||
|
||||
@staticmethod
|
||||
def is_text_generation_model(architectures: Union[str, List[str]]) -> bool:
|
||||
if isinstance(architectures, str):
|
||||
architectures = [architectures]
|
||||
if not architectures:
|
||||
logger.warning("No model architectures are specified")
|
||||
|
||||
is_txt_gen = partial(ModelRegistry._check_stateless,
|
||||
is_text_generation_model,
|
||||
default=False)
|
||||
|
||||
return any(is_txt_gen(arch) for arch in architectures)
|
||||
|
||||
@staticmethod
|
||||
def is_embedding_model(architectures: Union[str, List[str]]) -> bool:
|
||||
if isinstance(architectures, str):
|
||||
@@ -324,7 +340,11 @@ class ModelRegistry:
|
||||
if not architectures:
|
||||
logger.warning("No model architectures are specified")
|
||||
|
||||
return any(arch in _EMBEDDING_MODELS for arch in architectures)
|
||||
is_emb = partial(ModelRegistry._check_stateless,
|
||||
is_embedding_model,
|
||||
default=False)
|
||||
|
||||
return any(is_emb(arch) for arch in architectures)
|
||||
|
||||
@staticmethod
|
||||
def is_multimodal_model(architectures: Union[str, List[str]]) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user