[Model][0/N] Improve all pooling task | clean up (#25817)
Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
@@ -13,7 +13,10 @@ from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.models.config import VerifyAndUpdateConfig
|
||||
from vllm.transformers_utils.config import get_hf_file_bytes, get_hf_file_to_dict
|
||||
from vllm.transformers_utils.config import (
|
||||
get_hf_file_bytes,
|
||||
try_get_dense_modules,
|
||||
)
|
||||
|
||||
from .interfaces_base import VllmModelForPooling, is_pooling_model
|
||||
|
||||
@@ -35,43 +38,25 @@ _GENERATE_SUFFIXES = [
|
||||
def _load_st_projector(model_config: "ModelConfig") -> nn.Module | None:
|
||||
"""Load Sentence-Transformers Dense projection layers."""
|
||||
|
||||
dense_modules = try_get_dense_modules(
|
||||
model_config.model, revision=model_config.revision
|
||||
)
|
||||
|
||||
if dense_modules is None:
|
||||
return
|
||||
|
||||
try:
|
||||
modules = get_hf_file_to_dict(
|
||||
"modules.json", model_config.model, model_config.revision
|
||||
)
|
||||
if not modules:
|
||||
return None
|
||||
|
||||
if isinstance(modules, dict):
|
||||
modules = modules.get("modules", [])
|
||||
|
||||
dense_modules = [
|
||||
m for m in modules if m.get("type") == "sentence_transformers.models.Dense"
|
||||
]
|
||||
if not dense_modules:
|
||||
return None
|
||||
|
||||
layers = []
|
||||
for module in dense_modules:
|
||||
folder = module.get("path", "")
|
||||
|
||||
config_path = f"{folder}/config.json" if folder else "config.json"
|
||||
layer_config = get_hf_file_to_dict(
|
||||
config_path, model_config.model, model_config.revision
|
||||
)
|
||||
if not layer_config:
|
||||
continue
|
||||
|
||||
for layer_config in dense_modules:
|
||||
folder = layer_config["folder"]
|
||||
linear = nn.Linear(
|
||||
layer_config.get("in_features", 768),
|
||||
layer_config.get("out_features", 768),
|
||||
layer_config["in_features"],
|
||||
layer_config["out_features"],
|
||||
bias=layer_config.get("bias", True),
|
||||
dtype=model_config.head_dtype,
|
||||
)
|
||||
|
||||
if not _load_dense_weights(linear, folder, model_config):
|
||||
continue
|
||||
|
||||
layers.append(linear)
|
||||
if act_name := layer_config.get("activation_function"):
|
||||
layers.append(get_act_fn(act_name))
|
||||
@@ -303,18 +288,18 @@ def as_seq_cls_model(cls: _T) -> _T:
|
||||
from vllm.model_executor.models.interfaces import SupportsCrossEncoding
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .utils import get_model_hidden_size, maybe_prefix
|
||||
from .utils import maybe_prefix
|
||||
|
||||
class ModelForSequenceClassification(
|
||||
_create_pooling_model_cls(cls), SupportsCrossEncoding
|
||||
):
|
||||
def _init_pooler(self, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
config = vllm_config.model_config.hf_config
|
||||
model_config = vllm_config.model_config
|
||||
quant_config = vllm_config.quant_config
|
||||
hidden_size = get_model_hidden_size(config)
|
||||
|
||||
self.score = ReplicatedLinear(
|
||||
hidden_size,
|
||||
model_config.hidden_size,
|
||||
config.num_labels,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
|
||||
@@ -50,7 +50,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..layers.pooler import DispatchPooler, Pooler
|
||||
from .interfaces import SupportsPP
|
||||
from .interfaces import SupportsCrossEncoding, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
is_pp_missing_parameter,
|
||||
@@ -321,7 +321,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
class GPT2ForSequenceClassification(nn.Module):
|
||||
class GPT2ForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
"""GPT2 Model for sequence classification.
|
||||
|
||||
This class expands GPT2Model with pooling and score functions - last token
|
||||
@@ -358,6 +358,9 @@ class GPT2ForSequenceClassification(nn.Module):
|
||||
}
|
||||
)
|
||||
|
||||
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.transformer.get_input_embeddings(input_ids)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -148,37 +148,6 @@ class GritLMMeanPool(nn.Module):
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_len: torch.Tensor | None = None,
|
||||
instr_len: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert prompt_len is None or prompt_len == hidden_states.shape[0], (
|
||||
"partial prefill not supported with MEAN pooling"
|
||||
)
|
||||
|
||||
return hidden_states[instr_len:].mean(dim=0, dtype=torch.float32)
|
||||
|
||||
def forward_all(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
instr_lens: torch.Tensor,
|
||||
) -> list[torch.Tensor] | torch.Tensor:
|
||||
offset = 0
|
||||
pooled_data = list[torch.Tensor]()
|
||||
|
||||
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
|
||||
pooled_data.append(
|
||||
hidden_states[offset + instr_len : offset + prompt_len].mean(
|
||||
dim=0, dtype=torch.float32
|
||||
)
|
||||
)
|
||||
offset += prompt_len
|
||||
|
||||
return pooled_data
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor | list[torch.Tensor],
|
||||
@@ -190,18 +159,20 @@ class GritLMMeanPool(nn.Module):
|
||||
self._get_instruction_len(token_ids.cpu().numpy())
|
||||
for token_ids in get_prompt_token_ids(pooling_metadata)
|
||||
],
|
||||
device=prompt_lens.device,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
if isinstance(hidden_states, list):
|
||||
return [
|
||||
self.forward_one(h, prompt_len, instr_len)
|
||||
for h, prompt_len, instr_len in zip(
|
||||
hidden_states, prompt_lens, instr_lens
|
||||
offset = 0
|
||||
pooled_data = list[torch.Tensor]()
|
||||
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
|
||||
pooled_data.append(
|
||||
hidden_states[offset + instr_len : offset + prompt_len].mean(
|
||||
dim=0, dtype=torch.float32
|
||||
)
|
||||
]
|
||||
)
|
||||
offset += prompt_len
|
||||
|
||||
return self.forward_all(hidden_states, prompt_lens, instr_lens)
|
||||
return pooled_data
|
||||
|
||||
|
||||
class GritLMPooler(Pooler):
|
||||
|
||||
@@ -777,13 +777,6 @@ def fast_topk(
|
||||
return torch.topk(values, topk, dim=dim)
|
||||
|
||||
|
||||
def get_model_hidden_size(hf_config: PretrainedConfig) -> int:
|
||||
if hasattr(hf_config, "hidden_size"):
|
||||
return hf_config.hidden_size
|
||||
text_config = hf_config.get_text_config()
|
||||
return text_config.hidden_size
|
||||
|
||||
|
||||
# Chunk x along the num_tokens axis for sequence parallelism
|
||||
# NOTE: This is wrapped in a torch custom op to work around the following issue:
|
||||
# The output tensor can have a sequence length 0 at small input sequence lengths
|
||||
|
||||
Reference in New Issue
Block a user