[Doc] Update V1 status of various pooling models (#23189)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -20,7 +20,7 @@ from vllm.sequence import PoolerOutput
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
|
||||
|
||||
from .interfaces import SupportsV0Only
|
||||
from .interfaces import default_pooling_type
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -215,7 +215,8 @@ class GritLMPooler(Pooler):
|
||||
return build_output(pooled_data)
|
||||
|
||||
|
||||
class GritLM(LlamaForCausalLM, SupportsV0Only):
|
||||
@default_pooling_type("MEAN")
|
||||
class GritLM(LlamaForCausalLM):
|
||||
"""This class implements the embedding model for parasail-ai/GritLM-7B-vllm.
|
||||
|
||||
The class inherits from LlamaForCausalLM and provides a custom pooling
|
||||
@@ -241,7 +242,6 @@ class GritLM(LlamaForCausalLM, SupportsV0Only):
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# Use full attention for pooling (this is why V1 is not supported yet)
|
||||
if vllm_config.model_config.runner_type == "pooling":
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
hf_config.is_causal = False
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
from collections.abc import Iterable, Mapping, MutableSequence
|
||||
from typing import (TYPE_CHECKING, ClassVar, Literal, Optional, Protocol,
|
||||
Union, overload, runtime_checkable)
|
||||
TypeVar, Union, overload, runtime_checkable)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -641,11 +641,14 @@ def supports_cross_encoding(
|
||||
return is_pooling_model(model) and _supports_cross_encoding(model)
|
||||
|
||||
|
||||
def default_pooling_type(pooling_type: str) -> object:
|
||||
_T = TypeVar("_T", bound=type[torch.nn.Module])
|
||||
|
||||
|
||||
def default_pooling_type(pooling_type: str):
|
||||
"""Set default_pooling_type decorator. """
|
||||
|
||||
def func(model: object):
|
||||
model.default_pooling_type = pooling_type
|
||||
def func(model: _T) -> _T:
|
||||
model.default_pooling_type = pooling_type # type: ignore
|
||||
return model
|
||||
|
||||
return func
|
||||
|
||||
Reference in New Issue
Block a user