[Model][0/N] Improve all pooling task | clean up (#25817)

Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
wang.yuqi
2025-10-13 16:44:50 +08:00
committed by GitHub
parent 4f207c7174
commit 767c3ab869
19 changed files with 198 additions and 189 deletions
+18 -33
View File
@@ -13,7 +13,10 @@ from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.models.config import VerifyAndUpdateConfig
from vllm.transformers_utils.config import get_hf_file_bytes, get_hf_file_to_dict
from vllm.transformers_utils.config import (
get_hf_file_bytes,
try_get_dense_modules,
)
from .interfaces_base import VllmModelForPooling, is_pooling_model
@@ -35,43 +38,25 @@ _GENERATE_SUFFIXES = [
def _load_st_projector(model_config: "ModelConfig") -> nn.Module | None:
"""Load Sentence-Transformers Dense projection layers."""
dense_modules = try_get_dense_modules(
model_config.model, revision=model_config.revision
)
if dense_modules is None:
return
try:
modules = get_hf_file_to_dict(
"modules.json", model_config.model, model_config.revision
)
if not modules:
return None
if isinstance(modules, dict):
modules = modules.get("modules", [])
dense_modules = [
m for m in modules if m.get("type") == "sentence_transformers.models.Dense"
]
if not dense_modules:
return None
layers = []
for module in dense_modules:
folder = module.get("path", "")
config_path = f"{folder}/config.json" if folder else "config.json"
layer_config = get_hf_file_to_dict(
config_path, model_config.model, model_config.revision
)
if not layer_config:
continue
for layer_config in dense_modules:
folder = layer_config["folder"]
linear = nn.Linear(
layer_config.get("in_features", 768),
layer_config.get("out_features", 768),
layer_config["in_features"],
layer_config["out_features"],
bias=layer_config.get("bias", True),
dtype=model_config.head_dtype,
)
if not _load_dense_weights(linear, folder, model_config):
continue
layers.append(linear)
if act_name := layer_config.get("activation_function"):
layers.append(get_act_fn(act_name))
@@ -303,18 +288,18 @@ def as_seq_cls_model(cls: _T) -> _T:
from vllm.model_executor.models.interfaces import SupportsCrossEncoding
from vllm.sequence import IntermediateTensors
from .utils import get_model_hidden_size, maybe_prefix
from .utils import maybe_prefix
class ModelForSequenceClassification(
_create_pooling_model_cls(cls), SupportsCrossEncoding
):
def _init_pooler(self, vllm_config: "VllmConfig", prefix: str = ""):
config = vllm_config.model_config.hf_config
model_config = vllm_config.model_config
quant_config = vllm_config.quant_config
hidden_size = get_model_hidden_size(config)
self.score = ReplicatedLinear(
hidden_size,
model_config.hidden_size,
config.num_labels,
bias=False,
params_dtype=torch.float32,
+5 -2
View File
@@ -50,7 +50,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from ..layers.pooler import DispatchPooler, Pooler
from .interfaces import SupportsPP
from .interfaces import SupportsCrossEncoding, SupportsPP
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
@@ -321,7 +321,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
return loader.load_weights(weights)
class GPT2ForSequenceClassification(nn.Module):
class GPT2ForSequenceClassification(nn.Module, SupportsCrossEncoding):
"""GPT2 Model for sequence classification.
This class expands GPT2Model with pooling and score functions - last token
@@ -358,6 +358,9 @@ class GPT2ForSequenceClassification(nn.Module):
}
)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.transformer.get_input_embeddings(input_ids)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+10 -39
View File
@@ -148,37 +148,6 @@ class GritLMMeanPool(nn.Module):
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward_one(
self,
hidden_states: torch.Tensor,
prompt_len: torch.Tensor | None = None,
instr_len: torch.Tensor | None = None,
) -> torch.Tensor:
assert prompt_len is None or prompt_len == hidden_states.shape[0], (
"partial prefill not supported with MEAN pooling"
)
return hidden_states[instr_len:].mean(dim=0, dtype=torch.float32)
def forward_all(
self,
hidden_states: torch.Tensor,
prompt_lens: torch.Tensor,
instr_lens: torch.Tensor,
) -> list[torch.Tensor] | torch.Tensor:
offset = 0
pooled_data = list[torch.Tensor]()
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
pooled_data.append(
hidden_states[offset + instr_len : offset + prompt_len].mean(
dim=0, dtype=torch.float32
)
)
offset += prompt_len
return pooled_data
def forward(
self,
hidden_states: torch.Tensor | list[torch.Tensor],
@@ -190,18 +159,20 @@ class GritLMMeanPool(nn.Module):
self._get_instruction_len(token_ids.cpu().numpy())
for token_ids in get_prompt_token_ids(pooling_metadata)
],
device=prompt_lens.device,
device="cpu",
)
if isinstance(hidden_states, list):
return [
self.forward_one(h, prompt_len, instr_len)
for h, prompt_len, instr_len in zip(
hidden_states, prompt_lens, instr_lens
offset = 0
pooled_data = list[torch.Tensor]()
for prompt_len, instr_len in zip(prompt_lens, instr_lens):
pooled_data.append(
hidden_states[offset + instr_len : offset + prompt_len].mean(
dim=0, dtype=torch.float32
)
]
)
offset += prompt_len
return self.forward_all(hidden_states, prompt_lens, instr_lens)
return pooled_data
class GritLMPooler(Pooler):
-7
View File
@@ -777,13 +777,6 @@ def fast_topk(
return torch.topk(values, topk, dim=dim)
def get_model_hidden_size(hf_config: PretrainedConfig) -> int:
if hasattr(hf_config, "hidden_size"):
return hf_config.hidden_size
text_config = hf_config.get_text_config()
return text_config.hidden_size
# Chunk x along the num_tokens axis for sequence parallelism
# NOTE: This is wrapped in a torch custom op to work around the following issue:
# The output tensor can have a sequence length 0 at small input sequence lengths