feat: rename logit_bias/logit_scale to logit_mean/logit_sigma for affine score calibration (#39530)

Signed-off-by: Jesus Federico <jefp@amazon.com>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Jesus Federico
2026-04-13 04:43:44 +00:00
committed by GitHub
co-authored by Claude Opus 4.6 wang.yuqi
parent 2a3c32ce67
commit fa6ae31177
7 changed files with 86 additions and 35 deletions
+52 -5
View File
@@ -77,17 +77,31 @@ class PoolerConfig:
Defaults to None (i.e. set to max_model_len).
"""
## for classification models
## for classification models — affine score calibration
logit_mean: float | None = None
"""
If provided, subtract this value from classification logits before
activation. Used for affine score calibration (Platt scaling):
activation((logit - logit_mean) / logit_sigma). Defaults to None.
"""
logit_sigma: float | None = None
"""
If provided, divide the classification logits by this value after
mean subtraction. Used for affine score calibration (Platt scaling):
activation((logit - logit_mean) / logit_sigma). Defaults to None.
"""
# Deprecated aliases — will be removed in v0.21
logit_bias: float | None = None
"""
If provided, apply classification logit biases. Defaults to None.
Deprecated: Use logit_mean instead. Will be removed in v0.21.
"""
logit_scale: float | None = None
"""
If provided, scale the classification logits by this factor before
activation. Combined with logit_bias, enables affine score calibration:
activation(logit_scale * (score - logit_bias)). Defaults to None.
Deprecated: Use logit_sigma instead (note: logit_sigma = 1/logit_scale).
Will be removed in v0.21.
"""
## for reward models
@@ -105,6 +119,39 @@ class PoolerConfig:
"""
def __post_init__(self) -> None:
# Handle deprecated logit_bias → logit_mean
if self.logit_bias is not None:
if self.logit_mean is not None:
raise ValueError(
"Cannot set both `logit_bias` and `logit_mean`. "
"`logit_bias` is deprecated, use `logit_mean` instead."
)
logger.warning(
"`logit_bias` is deprecated and will be removed in v0.21. "
"Use `logit_mean` instead."
)
self.logit_mean = self.logit_bias
self.logit_bias = None
# Handle deprecated logit_scale → logit_sigma
if self.logit_scale is not None:
if self.logit_sigma is not None:
raise ValueError(
"Cannot set both `logit_scale` and `logit_sigma`. "
"`logit_scale` is deprecated, use `logit_sigma` instead."
)
logger.warning(
"`logit_scale` is deprecated and will be removed in v0.21. "
"Use `logit_sigma` instead (logit_sigma = 1/logit_scale)."
)
if self.logit_scale == 0:
raise ValueError("logit_scale cannot be 0 (division by zero)")
self.logit_sigma = 1.0 / self.logit_scale
self.logit_scale = None
if self.logit_sigma is not None and self.logit_sigma == 0:
raise ValueError("logit_sigma cannot be 0 (division by zero)")
if pooling_type := self.pooling_type:
if self.seq_pooling_type is not None:
raise ValueError(
@@ -103,16 +103,16 @@ class ClassifierPoolerHead(SequencePoolerHead):
def __init__(
self,
classifier: ClassifierFn | None = None,
logit_bias: float | None = None,
logit_scale: float | None = None,
logit_mean: float | None = None,
logit_sigma: float | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.classifier = classifier
self.logit_bias = logit_bias
self.logit_scale = logit_scale
self.logit_mean = logit_mean
self.logit_sigma = logit_sigma
self.head_dtype = head_dtype
self.activation = activation
@@ -140,10 +140,11 @@ class ClassifierPoolerHead(SequencePoolerHead):
logits = pooled_data
# logits shape: [batchsize, num_labels]
if self.logit_bias is not None:
logits -= self.logit_bias
if self.logit_scale is not None:
logits *= self.logit_scale
# Affine score calibration: activation((logit - mean) / sigma)
if self.logit_mean is not None:
logits = logits - self.logit_mean
if self.logit_sigma is not None:
logits = logits / self.logit_sigma
if self.activation is not None:
flags = [p.use_activation for p in pooling_params]
@@ -118,8 +118,8 @@ def pooler_for_classify(
head = ClassifierPoolerHead(
head_dtype=model_config.head_dtype,
classifier=classifier,
logit_bias=model_config.pooler_config.logit_bias,
logit_scale=model_config.pooler_config.logit_scale,
logit_mean=model_config.pooler_config.logit_mean,
logit_sigma=model_config.pooler_config.logit_sigma,
activation=resolve_classifier_act_fn(
model_config, static_num_labels=True, act_fn=act_fn
),
@@ -92,16 +92,16 @@ class TokenClassifierPoolerHead(TokenPoolerHead):
def __init__(
self,
classifier: ClassifierFn | None = None,
logit_bias: float | None = None,
logit_scale: float | None = None,
logit_mean: float | None = None,
logit_sigma: float | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.classifier = classifier
self.logit_bias = logit_bias
self.logit_scale = logit_scale
self.logit_mean = logit_mean
self.logit_sigma = logit_sigma
self.head_dtype = head_dtype
self.activation = activation
@@ -127,10 +127,11 @@ class TokenClassifierPoolerHead(TokenPoolerHead):
logits = pooled_data
# logits shape: [n_token, num_labels]
if self.logit_bias is not None:
logits -= self.logit_bias
if self.logit_scale is not None:
logits *= self.logit_scale
# Affine score calibration: activation((logit - mean) / sigma)
if self.logit_mean is not None:
logits = logits - self.logit_mean
if self.logit_sigma is not None:
logits = logits / self.logit_sigma
if self.activation is not None and pooling_param.use_activation:
logits = self.activation(logits)
@@ -127,8 +127,8 @@ def pooler_for_token_classify(
head = TokenClassifierPoolerHead(
head_dtype=model_config.head_dtype,
classifier=classifier,
logit_bias=model_config.pooler_config.logit_bias,
logit_scale=model_config.pooler_config.logit_scale,
logit_mean=model_config.pooler_config.logit_mean,
logit_sigma=model_config.pooler_config.logit_sigma,
activation=resolve_classifier_act_fn(
model_config, static_num_labels=False, act_fn=act_fn
),
+2 -2
View File
@@ -232,8 +232,8 @@ class JinaVLForSequenceClassificationConfig(VerifyAndUpdateConfig):
config = model_config.hf_config
config.num_labels = 1
pooler_config = model_config.pooler_config
if pooler_config.logit_bias is None:
pooler_config.logit_bias = 2.65
if pooler_config.logit_mean is None:
pooler_config.logit_mean = 2.65
class LlamaBidirectionalConfig(VerifyAndUpdateConfig):