[Bugfix] Honor STEP token pooling in DispatchPooler.for_seq_cls (#55307)
Signed-off-by: weihan <130119942+krsish@users.noreply.github.com>
This commit is contained in:
@@ -4,20 +4,25 @@
|
||||
# https://github.com/QwenLM/Qwen3-ASR
|
||||
|
||||
"""
|
||||
Online forced alignment example using Qwen3-ForcedAligner-0.6B.
|
||||
Online forced alignment example using Qwen3-ForcedAligner-0.6B-hf.
|
||||
|
||||
Forced alignment takes audio and reference text as input and produces
|
||||
word-level timestamps. The model predicts a time bin at each <timestamp>
|
||||
token position; multiplying by ``timestamp_segment_time`` gives milliseconds.
|
||||
|
||||
This example uses STEP pooling to return logits only for <timestamp> tokens.
|
||||
The ``step_tag_id`` value 151705 is this checkpoint's ``timestamp_token_id``
|
||||
from its config.json.
|
||||
|
||||
Start the server with:
|
||||
|
||||
vllm serve Qwen/Qwen3-ForcedAligner-0.6B \\
|
||||
--runner pooling \\
|
||||
--enforce-eager \\
|
||||
--trust-request-chat-template \\
|
||||
--hf-overrides \\
|
||||
'{"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]}'
|
||||
vllm serve Qwen/Qwen3-ForcedAligner-0.6B-hf \
|
||||
--runner pooling \
|
||||
--convert classify \
|
||||
--pooler-config \
|
||||
'{"tok_pooling_type":"STEP","step_tag_id":151705,"use_activation":false}' \
|
||||
--enforce-eager \
|
||||
--trust-request-chat-template
|
||||
|
||||
Then run:
|
||||
|
||||
@@ -108,7 +113,7 @@ def parse_response(response: requests.Response) -> dict[str, Any]:
|
||||
return result
|
||||
|
||||
|
||||
def load_timestamp_config(model: str) -> tuple[int, float]:
|
||||
def load_timestamp_segment_time(model: str) -> float:
|
||||
model_path = Path(model)
|
||||
config_path = (
|
||||
model_path / "config.json"
|
||||
@@ -119,7 +124,7 @@ def load_timestamp_config(model: str) -> tuple[int, float]:
|
||||
with config_path.open() as f:
|
||||
config = json.load(f)
|
||||
|
||||
return config["timestamp_token_id"], config["timestamp_segment_time"]
|
||||
return config["timestamp_segment_time"]
|
||||
|
||||
|
||||
def parse_args():
|
||||
@@ -129,7 +134,7 @@ def parse_args():
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="Qwen/Qwen3-ForcedAligner-0.6B",
|
||||
default="Qwen/Qwen3-ForcedAligner-0.6B-hf",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-path",
|
||||
@@ -147,8 +152,6 @@ def parse_args():
|
||||
|
||||
|
||||
def main(args):
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
api_url = f"http://{args.host}:{args.port}/pooling"
|
||||
prompt = build_prompt(args.words)
|
||||
audio_uri = (
|
||||
@@ -161,46 +164,22 @@ def main(args):
|
||||
pooling_response = post_http_request(payload=payload, api_url=api_url)
|
||||
result = parse_response(pooling_response)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model)
|
||||
timestamp_token_id, timestamp_segment_time = load_timestamp_config(args.model)
|
||||
|
||||
timestamp_segment_time = load_timestamp_segment_time(args.model)
|
||||
output = result["data"][0]
|
||||
logits = torch.tensor(output["data"])
|
||||
predictions = logits.argmax(dim=-1)
|
||||
token_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"]
|
||||
audio_pad_token_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
|
||||
|
||||
usage = result.get("usage") or {}
|
||||
prompt_tokens = usage.get("prompt_tokens")
|
||||
if prompt_tokens is not None and prompt_tokens != len(predictions):
|
||||
expected_timestamps = len(args.words) * 2
|
||||
if len(predictions) != expected_timestamps:
|
||||
raise RuntimeError(
|
||||
"The response length does not match the reported prompt token count."
|
||||
f"Expected {expected_timestamps} timestamp predictions, "
|
||||
f"but received {len(predictions)}. Check that the server was started "
|
||||
"with STEP pooling."
|
||||
)
|
||||
|
||||
try:
|
||||
audio_pad_index = token_ids.index(audio_pad_token_id)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("The prompt does not contain the audio pad token.") from exc
|
||||
|
||||
audio_token_shift = len(predictions) - len(token_ids)
|
||||
if audio_token_shift < 0:
|
||||
raise RuntimeError(
|
||||
"The response is shorter than the locally tokenized prompt. "
|
||||
"Check that the server was started with --trust-request-chat-template."
|
||||
)
|
||||
|
||||
ts_predictions = []
|
||||
for i, token_id in enumerate(token_ids):
|
||||
if token_id != timestamp_token_id:
|
||||
continue
|
||||
|
||||
prediction_index = i + audio_token_shift if i > audio_pad_index else i
|
||||
ts_predictions.append(
|
||||
predictions[prediction_index].item() * timestamp_segment_time
|
||||
)
|
||||
|
||||
if len(ts_predictions) < len(args.words) * 2:
|
||||
raise RuntimeError("The model did not return enough timestamp predictions.")
|
||||
ts_predictions = [
|
||||
prediction.item() * timestamp_segment_time for prediction in predictions
|
||||
]
|
||||
|
||||
for i, word in enumerate(args.words):
|
||||
start_ms = ts_predictions[i * 2]
|
||||
|
||||
@@ -3,12 +3,15 @@
|
||||
"""Unit tests for sequence and token pooling methods and their factories."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import PoolerConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.seqwise.methods import (
|
||||
CLSPool,
|
||||
@@ -16,6 +19,7 @@ from vllm.model_executor.layers.pooler.seqwise.methods import (
|
||||
MeanPool,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.special import DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import (
|
||||
AllPool,
|
||||
StepPool,
|
||||
@@ -300,6 +304,50 @@ class _FakeVllmConfig:
|
||||
scheduler_config: _FakeSchedulerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tok_pooling_type", ["ALL", "STEP"])
|
||||
def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type):
|
||||
"""Filter STEP rows before classification; preserve all rows for resolved ALL."""
|
||||
pooler_config = PoolerConfig(
|
||||
seq_pooling_type="LAST", tok_pooling_type=tok_pooling_type
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=_FakeSchedulerConfig(),
|
||||
model_config=SimpleNamespace(
|
||||
pooler_config=pooler_config,
|
||||
head_dtype=None,
|
||||
hf_config=PretrainedConfig(num_labels=3),
|
||||
),
|
||||
)
|
||||
classifier = torch.nn.Linear(4, 3)
|
||||
with set_current_vllm_config(config):
|
||||
pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=classifier)
|
||||
|
||||
token_ids = [[2, 99, 3, 99], [4, 99], [5, 6]]
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_classify",
|
||||
use_activation=False,
|
||||
step_tag_id=99 if tok_pooling_type == "STEP" else None,
|
||||
)
|
||||
for _ in token_ids
|
||||
]
|
||||
metadata = _make_metadata([4, 2, 2], token_ids=token_ids, pooling_params=params)
|
||||
hidden = torch.arange(32, dtype=torch.float32).reshape(8, 4)
|
||||
outputs = pooler(hidden, metadata)
|
||||
|
||||
expected_rows = (
|
||||
[[1, 3], [5], []]
|
||||
if tok_pooling_type == "STEP"
|
||||
else [[0, 1, 2, 3], [4, 5], [6, 7]]
|
||||
)
|
||||
assert len(outputs) == len(expected_rows)
|
||||
for output, rows in zip(outputs, expected_rows):
|
||||
torch.testing.assert_close(output, classifier(hidden[rows]))
|
||||
assert pooler.get_pooling_updates("token_classify").requires_token_ids == (
|
||||
tok_pooling_type == "STEP"
|
||||
)
|
||||
|
||||
|
||||
class TestAllPool:
|
||||
@staticmethod
|
||||
def _make_all_pool(
|
||||
|
||||
@@ -19,7 +19,7 @@ from .seqwise import (
|
||||
pooler_for_classify,
|
||||
pooler_for_embed,
|
||||
)
|
||||
from .tokwise import AllPool, pooler_for_token_classify, pooler_for_token_embed
|
||||
from .tokwise import pooler_for_token_classify, pooler_for_token_embed
|
||||
|
||||
|
||||
class DispatchPooler(Pooler):
|
||||
@@ -46,7 +46,6 @@ class DispatchPooler(Pooler):
|
||||
{
|
||||
"token_classify": pooler_for_token_classify(
|
||||
pooler_config,
|
||||
pooling=AllPool(),
|
||||
classifier=classifier,
|
||||
),
|
||||
"classify": pooler_for_classify(
|
||||
|
||||
Reference in New Issue
Block a user