[LoRA] Support variable num_labels for sequence classification (#57766)
Signed-off-by: linitra24 <renshuang.zhou@daocloud.io> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -171,16 +171,24 @@ class ClassifierPoolerHead(SequencePoolerHead):
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
|
||||
if self.classifier is not None:
|
||||
logits = self.classifier(pooled_data)
|
||||
logits: SequencePoolerHeadOutput = self.classifier(pooled_data)
|
||||
else:
|
||||
logits = pooled_data
|
||||
|
||||
# logits shape: [batchsize, num_labels]
|
||||
# Affine score calibration: activation((logit - mean) / sigma)
|
||||
if self.logit_mean is not None:
|
||||
logits = logits - self.logit_mean
|
||||
logits = (
|
||||
[value - self.logit_mean for value in logits]
|
||||
if isinstance(logits, list)
|
||||
else logits - self.logit_mean
|
||||
)
|
||||
if self.logit_sigma is not None:
|
||||
logits = logits / self.logit_sigma
|
||||
logits = (
|
||||
[value / self.logit_sigma for value in logits]
|
||||
if isinstance(logits, list)
|
||||
else logits / self.logit_sigma
|
||||
)
|
||||
|
||||
if self.activation is not None:
|
||||
flags = [p.use_activation for p in pooling_params]
|
||||
|
||||
@@ -130,7 +130,7 @@ def pooler_for_classify(
|
||||
logit_mean=model_config.pooler_config.logit_mean,
|
||||
logit_sigma=model_config.pooler_config.logit_sigma,
|
||||
activation=resolve_classifier_act_fn(
|
||||
model_config, static_num_labels=True, act_fn=act_fn
|
||||
model_config, static_num_labels=False, act_fn=act_fn
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user