[Bugfix] Resolve seq-cls num_labels from the top-level config for multimodal checkpoints (#50950)
Signed-off-by: Lazurite <43494437+Rapisurazurite@users.noreply.github.com>
This commit is contained in:
@@ -4,8 +4,12 @@
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
from transformers import Gemma3Config, PretrainedConfig, Qwen2Config
|
||||||
|
|
||||||
from vllm.model_executor.models.adapters import _create_pooling_model_cls
|
from vllm.model_executor.models.adapters import (
|
||||||
|
_create_pooling_model_cls,
|
||||||
|
_resolve_num_labels,
|
||||||
|
)
|
||||||
from vllm.model_executor.models.utils import AutoWeightsLoader, StageMissingLayer
|
from vllm.model_executor.models.utils import AutoWeightsLoader, StageMissingLayer
|
||||||
|
|
||||||
pytestmark = pytest.mark.cpu_test
|
pytestmark = pytest.mark.cpu_test
|
||||||
@@ -146,3 +150,45 @@ def test_pooling_load_weights_clones_probed_weights():
|
|||||||
expected = {n: p.data.clone() for n, p in ground_truth.named_parameters()}
|
expected = {n: p.data.clone() for n, p in ground_truth.named_parameters()}
|
||||||
|
|
||||||
_load_and_compare(_make_pooling_model(PackedWeightModel), ref, expected)
|
_load_and_compare(_make_pooling_model(PackedWeightModel), ref, expected)
|
||||||
|
|
||||||
|
|
||||||
|
def _composite_config(outer_labels=None, inner_labels=None):
|
||||||
|
"""Build a multimodal-style config whose text_config is a separate object."""
|
||||||
|
config = Gemma3Config(text_config={"num_hidden_layers": 1})
|
||||||
|
if outer_labels is not None:
|
||||||
|
config.num_labels = outer_labels
|
||||||
|
if inner_labels is not None:
|
||||||
|
config.get_text_config().num_labels = inner_labels
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_num_labels_text_only_config():
|
||||||
|
config = Qwen2Config(num_labels=7)
|
||||||
|
assert config.get_text_config() is config
|
||||||
|
assert _resolve_num_labels(config, config.get_text_config()) == 7
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_num_labels_defaults_when_undeclared():
|
||||||
|
config = _composite_config()
|
||||||
|
assert (
|
||||||
|
_resolve_num_labels(config, config.get_text_config())
|
||||||
|
== PretrainedConfig().num_labels
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_num_labels_declared_on_outer_config():
|
||||||
|
"""Multimodal checkpoints keep id2label/problem_type on the top-level config."""
|
||||||
|
config = _composite_config(outer_labels=20)
|
||||||
|
assert config.get_text_config().num_labels == PretrainedConfig().num_labels
|
||||||
|
assert _resolve_num_labels(config, config.get_text_config()) == 20
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_num_labels_declared_on_text_config():
|
||||||
|
"""Overrides written into text_config keep working."""
|
||||||
|
config = _composite_config(inner_labels=5)
|
||||||
|
assert _resolve_num_labels(config, config.get_text_config()) == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_num_labels_outer_wins_when_both_declared():
|
||||||
|
config = _composite_config(outer_labels=20, inner_labels=5)
|
||||||
|
assert _resolve_num_labels(config, config.get_text_config()) == 20
|
||||||
|
|||||||
@@ -261,6 +261,29 @@ def as_embedding_model(cls: _T) -> _T:
|
|||||||
return ModelForEmbedding # type: ignore
|
return ModelForEmbedding # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_num_labels(hf_config: Any, text_config: Any) -> int:
|
||||||
|
"""Resolve the label count for a sequence classification head.
|
||||||
|
|
||||||
|
``PretrainedConfig.num_labels`` is derived from ``id2label``, which always
|
||||||
|
carries a default of two entries. Composite configs (such as multimodal
|
||||||
|
checkpoints) declare their label space on the top-level config, so reading
|
||||||
|
``num_labels`` from ``get_text_config()`` silently returns that default and
|
||||||
|
builds a score head of the wrong size.
|
||||||
|
|
||||||
|
Prefer the top-level config whenever it declares a label space of its own,
|
||||||
|
mirroring the ``classifier_from_token`` / ``method`` lookups in
|
||||||
|
``as_seq_cls_model``.
|
||||||
|
"""
|
||||||
|
if text_config is hf_config:
|
||||||
|
return hf_config.num_labels
|
||||||
|
|
||||||
|
from transformers import PretrainedConfig
|
||||||
|
|
||||||
|
if hf_config.num_labels != PretrainedConfig().num_labels:
|
||||||
|
return hf_config.num_labels
|
||||||
|
return text_config.num_labels
|
||||||
|
|
||||||
|
|
||||||
def as_seq_cls_model(cls: _T) -> _T:
|
def as_seq_cls_model(cls: _T) -> _T:
|
||||||
"""
|
"""
|
||||||
Subclass an existing vLLM model to support classify and score tasks.
|
Subclass an existing vLLM model to support classify and score tasks.
|
||||||
@@ -320,7 +343,7 @@ def as_seq_cls_model(cls: _T) -> _T:
|
|||||||
|
|
||||||
self.score = ReplicatedLinear(
|
self.score = ReplicatedLinear(
|
||||||
model_config.get_hidden_size(),
|
model_config.get_hidden_size(),
|
||||||
text_config.num_labels,
|
_resolve_num_labels(hf_config, text_config),
|
||||||
bias=False,
|
bias=False,
|
||||||
params_dtype=model_config.head_dtype,
|
params_dtype=model_config.head_dtype,
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
|
|||||||
Reference in New Issue
Block a user