[Model] Add Clef-Flash support (#61308)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Codex
Isotr0py
parent
0d2689e0df
commit
380638afbb
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Clef-Flash: the Qwen3.5 backbone with a joint schema decision head."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .clef_joint_schema import JOINT_HEAD_WEIGHTS, JointSchemaHead, JointSchemaPooler
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .qwen3_5 import Qwen3_5ForConditionalGeneration, Qwen3_5ProcessingInfo
|
||||
from .qwen3_vl import Qwen3VLDummyInputsBuilder, Qwen3VLMultiModalProcessor
|
||||
|
||||
|
||||
@default_pooling_type(tok_pooling_type="ALL")
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Qwen3VLMultiModalProcessor,
|
||||
info=Qwen3_5ProcessingInfo,
|
||||
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
||||
)
|
||||
class ClefFlashForJointSchema(Qwen3_5ForConditionalGeneration):
|
||||
is_pooling_model = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
model_config = vllm_config.model_config
|
||||
self.pooler = JointSchemaPooler(
|
||||
JointSchemaHead(**model_config.hf_config.joint_head_config),
|
||||
self._embed_lexical,
|
||||
)
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=model_config.model,
|
||||
revision=model_config.revision,
|
||||
prefix="pooler.head.",
|
||||
allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
|
||||
)
|
||||
]
|
||||
|
||||
def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
|
||||
@@ -0,0 +1,343 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared joint schema head for Clef decision models.
|
||||
|
||||
The head reads the final hidden states of the whole prompt, so it runs as a
|
||||
``token_classify`` pooler.
|
||||
|
||||
Each request carries its schema in ``PoolingParams.extra_kwargs`` under
|
||||
``"joint_schema"``:
|
||||
|
||||
- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question.
|
||||
- ``question_spans``: the ``[start, end)`` of each question's instructions.
|
||||
- ``option_spans``: the ``[start, end)`` of each option, question by question.
|
||||
- ``option_counts``: the number of options of each question.
|
||||
|
||||
Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``),
|
||||
so they hold after multimodal placeholders are expanded. The output has one
|
||||
value per option, in ``option_spans`` order: its probability within its
|
||||
question, or its logit with ``use_activation=False``.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Set
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import AllPool
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import PoolerOutput
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
JOINT_HEAD_WEIGHTS = "joint_head.safetensors"
|
||||
NUM_QUESTION_TYPES = 3
|
||||
MAX_LOGIT_SCALE = math.log(100.0)
|
||||
|
||||
|
||||
def segment_mean(
|
||||
values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32)
|
||||
sums.index_add_(0, segment_ids, values.float())
|
||||
return (sums / lengths[:, None]).to(values.dtype)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaIndices:
|
||||
"""Where one prompt's questions and options are, on the model's device."""
|
||||
|
||||
positions: torch.Tensor
|
||||
"""Every token of every span, questions first."""
|
||||
span_ids: torch.Tensor
|
||||
"""The span of each of ``positions``."""
|
||||
span_lengths: torch.Tensor
|
||||
"""``[Q + O]``."""
|
||||
question_types: torch.Tensor
|
||||
"""``[Q]``."""
|
||||
option_question: torch.Tensor
|
||||
"""The question of each option, ``[O]``."""
|
||||
option_slots: torch.Tensor
|
||||
"""Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``."""
|
||||
option_mask: torch.Tensor
|
||||
"""``[Q, max_options]``, true at the slots of options."""
|
||||
|
||||
|
||||
class EvidenceRoutingLayer(nn.Module):
|
||||
def __init__(self, width: int, heads: int, feedforward: int) -> None:
|
||||
super().__init__()
|
||||
self.query_norm = nn.LayerNorm(width)
|
||||
self.memory_norm = nn.LayerNorm(width)
|
||||
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
|
||||
self.feedforward_norm = nn.LayerNorm(width)
|
||||
# The Identity takes the place of the checkpoint's dropout.
|
||||
self.feedforward = nn.Sequential(
|
||||
nn.Linear(width, feedforward),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(feedforward, width),
|
||||
)
|
||||
|
||||
def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
|
||||
# Separate key and value tensors, like the reference, keep attention's
|
||||
# unpacked input projections.
|
||||
routed, _ = self.attention(
|
||||
self.query_norm(queries),
|
||||
self.memory_norm(memory),
|
||||
self.memory_norm(memory),
|
||||
need_weights=False,
|
||||
)
|
||||
queries = queries + routed
|
||||
return queries + self.feedforward(self.feedforward_norm(queries))
|
||||
|
||||
|
||||
class JointSchemaHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
width: int,
|
||||
routing_layers: int,
|
||||
layers: int,
|
||||
heads: int,
|
||||
feedforward: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = nn.LayerNorm(hidden_size)
|
||||
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.global_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width)
|
||||
self.evidence_layers = nn.ModuleList(
|
||||
EvidenceRoutingLayer(width, heads, feedforward)
|
||||
for _ in range(routing_layers)
|
||||
)
|
||||
self.option_summary_norm = nn.LayerNorm(width)
|
||||
self.layers = nn.ModuleList(
|
||||
nn.TransformerDecoderLayer(
|
||||
d_model=width,
|
||||
nhead=heads,
|
||||
dim_feedforward=feedforward,
|
||||
dropout=0.0,
|
||||
activation="gelu",
|
||||
batch_first=True,
|
||||
norm_first=True,
|
||||
)
|
||||
for _ in range(layers)
|
||||
)
|
||||
self.field_norm = nn.LayerNorm(width)
|
||||
self.option_norm = nn.LayerNorm(width)
|
||||
self.residual_scorer = nn.Sequential(
|
||||
nn.Linear(width * 4, width),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(width, 1),
|
||||
)
|
||||
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.residual_gate = nn.Parameter(torch.zeros(()))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
lexical: torch.Tensor,
|
||||
schema: SchemaIndices,
|
||||
) -> torch.Tensor:
|
||||
"""The ``[O]`` option logits of one prompt, from its ``[L, hidden]``
|
||||
final hidden states and each option's ``[O, hidden]`` mean output
|
||||
embedding."""
|
||||
num_questions = schema.question_types.shape[0]
|
||||
option_question = schema.option_question
|
||||
hidden_states = self.hidden_norm(hidden_states)
|
||||
memory = self.memory_projection(hidden_states).unsqueeze(0)
|
||||
global_vector = hidden_states[-1]
|
||||
means = segment_mean(
|
||||
hidden_states[schema.positions], schema.span_ids, schema.span_lengths
|
||||
)
|
||||
question_vectors, contexts = means[:num_questions], means[num_questions:]
|
||||
|
||||
options = (
|
||||
self.option_context_projection(contexts)
|
||||
+ self.option_lexical_projection(lexical)
|
||||
+ self.option_question_projection(question_vectors)[option_question]
|
||||
).unsqueeze(0)
|
||||
for layer in self.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
options = options[0]
|
||||
|
||||
# Each field summarizes its options, weighted by their match to it.
|
||||
base_fields = self.question_projection(question_vectors)
|
||||
width = options.shape[-1]
|
||||
grid = options.new_zeros(schema.option_mask.numel(), width)
|
||||
grid[schema.option_slots] = options
|
||||
grid = grid.view(*schema.option_mask.shape, width)
|
||||
routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width)
|
||||
routing = routing.masked_fill(~schema.option_mask, float("-inf"))
|
||||
summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid)
|
||||
fields = (
|
||||
base_fields
|
||||
+ self.option_summary_norm(summaries)
|
||||
+ self.global_projection(global_vector).unsqueeze(0)
|
||||
+ self.type_embedding(schema.question_types)
|
||||
).unsqueeze(0)
|
||||
for layer in self.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = self.field_norm(fields[0])[option_question]
|
||||
|
||||
anchors = F.normalize(question_vectors + global_vector, dim=-1)
|
||||
prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
prior = prior_scale * (
|
||||
F.normalize(lexical, dim=-1) * anchors[option_question]
|
||||
).sum(dim=-1)
|
||||
options = self.option_norm(options)
|
||||
cosine = F.cosine_similarity(fields, options, dim=-1)
|
||||
features = torch.cat(
|
||||
[fields, options, fields * options, torch.abs(fields - options)], dim=-1
|
||||
)
|
||||
residual = self.residual_scorer(features).squeeze(-1)
|
||||
joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
joint = joint_scale * cosine + residual
|
||||
return prior + torch.sigmoid(self.residual_gate) * joint
|
||||
|
||||
|
||||
def parse_joint_schema(
|
||||
extra_kwargs: dict[str, Any] | None, prompt_len: int
|
||||
) -> tuple[list[int], list[tuple[int, int]], list[int]] | None:
|
||||
"""The question types, the absolute spans of the questions then of the
|
||||
options, and the option counts. None if the schema is missing or invalid."""
|
||||
schema = (extra_kwargs or {}).get("joint_schema")
|
||||
if not isinstance(schema, dict):
|
||||
return None
|
||||
try:
|
||||
types = [int(t) for t in schema["question_types"]]
|
||||
counts = [int(c) for c in schema["option_counts"]]
|
||||
question_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["question_spans"]
|
||||
]
|
||||
option_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["option_spans"]
|
||||
]
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
spans = question_spans + option_spans
|
||||
if (
|
||||
not types
|
||||
or len(counts) != len(types)
|
||||
or len(question_spans) != len(types)
|
||||
or len(option_spans) != sum(counts)
|
||||
or any(not 0 <= t < NUM_QUESTION_TYPES for t in types)
|
||||
or any(c < 1 for c in counts)
|
||||
or any(not 0 <= start < end <= prompt_len for start, end in spans)
|
||||
):
|
||||
return None
|
||||
return types, spans, counts
|
||||
|
||||
|
||||
class JointSchemaPooler(Pooler):
|
||||
"""Runs the joint schema head on each finished prompt.
|
||||
|
||||
It never raises, because it runs in the engine core: a prompt without a
|
||||
valid schema, such as a profiling dummy, gets an empty output.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head: JointSchemaHead,
|
||||
embed_lexical: Callable[[torch.Tensor], torch.Tensor],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.pooling = AllPool()
|
||||
self.head = head
|
||||
self.embed_lexical = embed_lexical
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_classify"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled = self.pooling(hidden_states, pooling_metadata)
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
outputs: list[torch.Tensor | None] = []
|
||||
for hidden, token_ids, params in zip(
|
||||
pooled, prompt_token_ids, pooling_metadata.pooling_params
|
||||
):
|
||||
if hidden is None:
|
||||
outputs.append(None)
|
||||
continue
|
||||
parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0])
|
||||
if parsed is None:
|
||||
outputs.append(hidden.new_empty(0, dtype=torch.float32))
|
||||
continue
|
||||
schema, lexical_ids, lexical_span_ids = self._indices(
|
||||
*parsed, hidden.device
|
||||
)
|
||||
lexical = segment_mean(
|
||||
self.embed_lexical(
|
||||
async_tensor_h2d(token_ids[lexical_ids], hidden.device)
|
||||
),
|
||||
lexical_span_ids,
|
||||
schema.span_lengths[len(parsed[0]) :],
|
||||
)
|
||||
logits = self.head(hidden, lexical, schema).float()
|
||||
if params.use_activation is not False:
|
||||
mask = schema.option_mask
|
||||
grid = logits.new_full((mask.numel(),), float("-inf"))
|
||||
grid[schema.option_slots] = logits
|
||||
probs = grid.view(mask.shape).softmax(dim=-1)
|
||||
logits = probs.view(-1)[schema.option_slots]
|
||||
outputs.append(logits)
|
||||
return outputs
|
||||
|
||||
@staticmethod
|
||||
def _indices(
|
||||
types: list[int],
|
||||
spans: list[tuple[int, int]],
|
||||
counts: list[int],
|
||||
device: torch.device,
|
||||
) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]:
|
||||
"""The schema's indices, plus the option tokens' positions (on the CPU)
|
||||
and their option spans, for the lexical embeddings."""
|
||||
num_questions = len(types)
|
||||
lengths = torch.tensor([end - start for start, end in spans])
|
||||
span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths)
|
||||
positions = torch.cat([torch.arange(start, end) for start, end in spans])
|
||||
num_question_tokens = int(lengths[:num_questions].sum())
|
||||
|
||||
counts_cpu = torch.tensor(counts)
|
||||
max_options = max(counts)
|
||||
option_question = torch.repeat_interleave(
|
||||
torch.arange(num_questions), counts_cpu
|
||||
)
|
||||
option_slots = torch.cat(
|
||||
[torch.arange(count) + q * max_options for q, count in enumerate(counts)]
|
||||
)
|
||||
schema = SchemaIndices(
|
||||
positions=async_tensor_h2d(positions, device),
|
||||
span_ids=async_tensor_h2d(span_ids, device),
|
||||
span_lengths=async_tensor_h2d(lengths, device),
|
||||
question_types=async_tensor_h2d(types, device, dtype=torch.long),
|
||||
option_question=async_tensor_h2d(option_question, device),
|
||||
option_slots=async_tensor_h2d(option_slots, device),
|
||||
option_mask=async_tensor_h2d(
|
||||
torch.arange(max_options) < counts_cpu[:, None], device
|
||||
),
|
||||
)
|
||||
lexical_span_ids = async_tensor_h2d(
|
||||
span_ids[num_question_tokens:] - num_questions, device
|
||||
)
|
||||
return schema, positions[num_question_tokens:], lexical_span_ids
|
||||
@@ -1,47 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Clef-Omni (https://huggingface.co/Cloudflare/clef-omni).
|
||||
|
||||
The Qwen3-Omni thinker with a joint schema head that scores every option of
|
||||
every question in one forward pass. The head reads the final hidden states of
|
||||
the whole prompt, so it runs as a ``token_classify`` pooler.
|
||||
|
||||
Each request carries its schema in ``PoolingParams.extra_kwargs`` under
|
||||
``"joint_schema"``:
|
||||
|
||||
- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question.
|
||||
- ``question_spans``: the ``[start, end)`` of each question's instructions.
|
||||
- ``option_spans``: the ``[start, end)`` of each option, question by question.
|
||||
- ``option_counts``: the number of options of each question.
|
||||
|
||||
Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``),
|
||||
so they hold after multimodal placeholders are expanded. The output has one
|
||||
value per option, in ``option_spans`` order: its probability within its
|
||||
question, or its logit with ``use_activation=False``.
|
||||
"""
|
||||
"""The Qwen3-Omni thinker with a Clef joint schema head."""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Mapping, Set
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import AllPool
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import PoolerOutput
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .clef_joint_schema import JOINT_HEAD_WEIGHTS, JointSchemaHead, JointSchemaPooler
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerDummyInputsBuilder,
|
||||
@@ -50,314 +25,6 @@ from .qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerProcessingInfo,
|
||||
)
|
||||
|
||||
JOINT_HEAD_WEIGHTS = "joint_head.safetensors"
|
||||
NUM_QUESTION_TYPES = 3
|
||||
MAX_LOGIT_SCALE = math.log(100.0)
|
||||
|
||||
|
||||
def segment_mean(
|
||||
values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32)
|
||||
sums.index_add_(0, segment_ids, values.float())
|
||||
return (sums / lengths[:, None]).to(values.dtype)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaIndices:
|
||||
"""Where one prompt's questions and options are, on the model's device."""
|
||||
|
||||
positions: torch.Tensor
|
||||
"""Every token of every span, questions first."""
|
||||
span_ids: torch.Tensor
|
||||
"""The span of each of ``positions``."""
|
||||
span_lengths: torch.Tensor
|
||||
"""``[Q + O]``."""
|
||||
question_types: torch.Tensor
|
||||
"""``[Q]``."""
|
||||
option_question: torch.Tensor
|
||||
"""The question of each option, ``[O]``."""
|
||||
option_slots: torch.Tensor
|
||||
"""Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``."""
|
||||
option_mask: torch.Tensor
|
||||
"""``[Q, max_options]``, true at the slots of options."""
|
||||
|
||||
|
||||
class EvidenceRoutingLayer(nn.Module):
|
||||
def __init__(self, width: int, heads: int, feedforward: int) -> None:
|
||||
super().__init__()
|
||||
self.query_norm = nn.LayerNorm(width)
|
||||
self.memory_norm = nn.LayerNorm(width)
|
||||
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
|
||||
self.feedforward_norm = nn.LayerNorm(width)
|
||||
# The Identity takes the place of the checkpoint's dropout.
|
||||
self.feedforward = nn.Sequential(
|
||||
nn.Linear(width, feedforward),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(feedforward, width),
|
||||
)
|
||||
|
||||
def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
|
||||
# Separate key and value tensors, like the reference, keep attention's
|
||||
# unpacked input projections.
|
||||
routed, _ = self.attention(
|
||||
self.query_norm(queries),
|
||||
self.memory_norm(memory),
|
||||
self.memory_norm(memory),
|
||||
need_weights=False,
|
||||
)
|
||||
queries = queries + routed
|
||||
return queries + self.feedforward(self.feedforward_norm(queries))
|
||||
|
||||
|
||||
class JointSchemaHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
width: int,
|
||||
routing_layers: int,
|
||||
layers: int,
|
||||
heads: int,
|
||||
feedforward: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = nn.LayerNorm(hidden_size)
|
||||
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.global_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width)
|
||||
self.evidence_layers = nn.ModuleList(
|
||||
EvidenceRoutingLayer(width, heads, feedforward)
|
||||
for _ in range(routing_layers)
|
||||
)
|
||||
self.option_summary_norm = nn.LayerNorm(width)
|
||||
self.layers = nn.ModuleList(
|
||||
nn.TransformerDecoderLayer(
|
||||
d_model=width,
|
||||
nhead=heads,
|
||||
dim_feedforward=feedforward,
|
||||
dropout=0.0,
|
||||
activation="gelu",
|
||||
batch_first=True,
|
||||
norm_first=True,
|
||||
)
|
||||
for _ in range(layers)
|
||||
)
|
||||
self.field_norm = nn.LayerNorm(width)
|
||||
self.option_norm = nn.LayerNorm(width)
|
||||
self.residual_scorer = nn.Sequential(
|
||||
nn.Linear(width * 4, width),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(width, 1),
|
||||
)
|
||||
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.residual_gate = nn.Parameter(torch.zeros(()))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
lexical: torch.Tensor,
|
||||
schema: SchemaIndices,
|
||||
) -> torch.Tensor:
|
||||
"""The ``[O]`` option logits of one prompt, from its ``[L, hidden]``
|
||||
final hidden states and each option's ``[O, hidden]`` mean output
|
||||
embedding."""
|
||||
num_questions = schema.question_types.shape[0]
|
||||
option_question = schema.option_question
|
||||
hidden_states = self.hidden_norm(hidden_states)
|
||||
memory = self.memory_projection(hidden_states).unsqueeze(0)
|
||||
global_vector = hidden_states[-1]
|
||||
means = segment_mean(
|
||||
hidden_states[schema.positions], schema.span_ids, schema.span_lengths
|
||||
)
|
||||
question_vectors, contexts = means[:num_questions], means[num_questions:]
|
||||
|
||||
options = (
|
||||
self.option_context_projection(contexts)
|
||||
+ self.option_lexical_projection(lexical)
|
||||
+ self.option_question_projection(question_vectors)[option_question]
|
||||
).unsqueeze(0)
|
||||
for layer in self.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
options = options[0]
|
||||
|
||||
# Each field summarizes its options, weighted by their match to it.
|
||||
base_fields = self.question_projection(question_vectors)
|
||||
width = options.shape[-1]
|
||||
grid = options.new_zeros(schema.option_mask.numel(), width)
|
||||
grid[schema.option_slots] = options
|
||||
grid = grid.view(*schema.option_mask.shape, width)
|
||||
routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width)
|
||||
routing = routing.masked_fill(~schema.option_mask, float("-inf"))
|
||||
summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid)
|
||||
fields = (
|
||||
base_fields
|
||||
+ self.option_summary_norm(summaries)
|
||||
+ self.global_projection(global_vector).unsqueeze(0)
|
||||
+ self.type_embedding(schema.question_types)
|
||||
).unsqueeze(0)
|
||||
for layer in self.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = self.field_norm(fields[0])[option_question]
|
||||
|
||||
anchors = F.normalize(question_vectors + global_vector, dim=-1)
|
||||
prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
prior = prior_scale * (
|
||||
F.normalize(lexical, dim=-1) * anchors[option_question]
|
||||
).sum(dim=-1)
|
||||
options = self.option_norm(options)
|
||||
cosine = F.cosine_similarity(fields, options, dim=-1)
|
||||
features = torch.cat(
|
||||
[fields, options, fields * options, torch.abs(fields - options)], dim=-1
|
||||
)
|
||||
residual = self.residual_scorer(features).squeeze(-1)
|
||||
joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
joint = joint_scale * cosine + residual
|
||||
return prior + torch.sigmoid(self.residual_gate) * joint
|
||||
|
||||
|
||||
def parse_joint_schema(
|
||||
extra_kwargs: dict[str, Any] | None, prompt_len: int
|
||||
) -> tuple[list[int], list[tuple[int, int]], list[int]] | None:
|
||||
"""The question types, the absolute spans of the questions then of the
|
||||
options, and the option counts. None if the schema is missing or invalid."""
|
||||
schema = (extra_kwargs or {}).get("joint_schema")
|
||||
if not isinstance(schema, dict):
|
||||
return None
|
||||
try:
|
||||
types = [int(t) for t in schema["question_types"]]
|
||||
counts = [int(c) for c in schema["option_counts"]]
|
||||
question_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["question_spans"]
|
||||
]
|
||||
option_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["option_spans"]
|
||||
]
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
spans = question_spans + option_spans
|
||||
if (
|
||||
not types
|
||||
or len(counts) != len(types)
|
||||
or len(question_spans) != len(types)
|
||||
or len(option_spans) != sum(counts)
|
||||
or any(not 0 <= t < NUM_QUESTION_TYPES for t in types)
|
||||
or any(c < 1 for c in counts)
|
||||
or any(not 0 <= start < end <= prompt_len for start, end in spans)
|
||||
):
|
||||
return None
|
||||
return types, spans, counts
|
||||
|
||||
|
||||
class JointSchemaPooler(Pooler):
|
||||
"""Runs the joint schema head on each finished prompt.
|
||||
|
||||
It never raises, because it runs in the engine core: a prompt without a
|
||||
valid schema, such as a profiling dummy, gets an empty output.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head: JointSchemaHead,
|
||||
embed_lexical: Callable[[torch.Tensor], torch.Tensor],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.pooling = AllPool()
|
||||
self.head = head
|
||||
self.embed_lexical = embed_lexical
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_classify"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled = self.pooling(hidden_states, pooling_metadata)
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
outputs: list[torch.Tensor | None] = []
|
||||
for hidden, token_ids, params in zip(
|
||||
pooled, prompt_token_ids, pooling_metadata.pooling_params
|
||||
):
|
||||
if hidden is None:
|
||||
outputs.append(None)
|
||||
continue
|
||||
parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0])
|
||||
if parsed is None:
|
||||
outputs.append(hidden.new_empty(0, dtype=torch.float32))
|
||||
continue
|
||||
schema, lexical_ids, lexical_span_ids = self._indices(
|
||||
*parsed, hidden.device
|
||||
)
|
||||
lexical = segment_mean(
|
||||
self.embed_lexical(
|
||||
async_tensor_h2d(token_ids[lexical_ids], hidden.device)
|
||||
),
|
||||
lexical_span_ids,
|
||||
schema.span_lengths[len(parsed[0]) :],
|
||||
)
|
||||
logits = self.head(hidden, lexical, schema).float()
|
||||
if params.use_activation is not False:
|
||||
mask = schema.option_mask
|
||||
grid = logits.new_full((mask.numel(),), float("-inf"))
|
||||
grid[schema.option_slots] = logits
|
||||
probs = grid.view(mask.shape).softmax(dim=-1)
|
||||
logits = probs.view(-1)[schema.option_slots]
|
||||
outputs.append(logits)
|
||||
return outputs
|
||||
|
||||
@staticmethod
|
||||
def _indices(
|
||||
types: list[int],
|
||||
spans: list[tuple[int, int]],
|
||||
counts: list[int],
|
||||
device: torch.device,
|
||||
) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]:
|
||||
"""The schema's indices, plus the option tokens' positions (on the CPU)
|
||||
and their option spans, for the lexical embeddings."""
|
||||
num_questions = len(types)
|
||||
lengths = torch.tensor([end - start for start, end in spans])
|
||||
span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths)
|
||||
positions = torch.cat([torch.arange(start, end) for start, end in spans])
|
||||
num_question_tokens = int(lengths[:num_questions].sum())
|
||||
|
||||
counts_cpu = torch.tensor(counts)
|
||||
max_options = max(counts)
|
||||
option_question = torch.repeat_interleave(
|
||||
torch.arange(num_questions), counts_cpu
|
||||
)
|
||||
option_slots = torch.cat(
|
||||
[torch.arange(count) + q * max_options for q, count in enumerate(counts)]
|
||||
)
|
||||
schema = SchemaIndices(
|
||||
positions=async_tensor_h2d(positions, device),
|
||||
span_ids=async_tensor_h2d(span_ids, device),
|
||||
span_lengths=async_tensor_h2d(lengths, device),
|
||||
question_types=async_tensor_h2d(types, device, dtype=torch.long),
|
||||
option_question=async_tensor_h2d(option_question, device),
|
||||
option_slots=async_tensor_h2d(option_slots, device),
|
||||
option_mask=async_tensor_h2d(
|
||||
torch.arange(max_options) < counts_cpu[:, None], device
|
||||
),
|
||||
)
|
||||
lexical_span_ids = async_tensor_h2d(
|
||||
span_ids[num_question_tokens:] - num_questions, device
|
||||
)
|
||||
return schema, positions[num_question_tokens:], lexical_span_ids
|
||||
|
||||
|
||||
MAX_FRAME_PIXELS = 256 * 32 * 32
|
||||
|
||||
|
||||
|
||||
@@ -287,6 +287,7 @@ _REWARD_MODELS = {
|
||||
|
||||
_TOKEN_CLASSIFICATION_MODELS = {
|
||||
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
|
||||
"ClefFlashForJointSchema": ("clef_flash", "ClefFlashForJointSchema"),
|
||||
"ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
|
||||
"ModernBertForTokenClassification": (
|
||||
"modernbert",
|
||||
|
||||
Reference in New Issue
Block a user