From 12f3f19c1959174c81735b4d7dbfac7182a5e3c7 Mon Sep 17 00:00:00 2001 From: Nathan Price <125999937+TheCodeWrangler@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:54:59 -0500 Subject: [PATCH] feat(qwen3-asr): support prompt parameter in v1/audio/transcriptions (#35415) Signed-off-by: Nathan Price Co-authored-by: Cursor Co-authored-by: Cyrus Leung Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../openai/openai_transcription_client.py | 28 +++++++- .../test_qwen3_asr_sanitize_prompt.py | 64 ++++++++++++++++++ vllm/model_executor/models/qwen3_asr.py | 66 +++++++++++++++---- 3 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py diff --git a/examples/speech_to_text/openai/openai_transcription_client.py b/examples/speech_to_text/openai/openai_transcription_client.py index 396edba115..f928c06d45 100644 --- a/examples/speech_to_text/openai/openai_transcription_client.py +++ b/examples/speech_to_text/openai/openai_transcription_client.py @@ -33,15 +33,23 @@ def sync_openai( *, repetition_penalty: float = 1.3, hotwords: str = None, + prompt: str | None = None, ): """ Perform synchronous transcription using OpenAI-compatible API. + + The optional ``prompt`` is the OpenAI-API ``prompt`` field (style / + vocabulary hint). It is wired through model-by-model: Whisper uses it + as a ``<|prev|>`` continuation hint, Qwen3-ASR maps it into the + chat-template ``system`` turn. Models that do not consume it accept + it without effect. """ with open(audio_path, "rb") as f: transcription = client.audio.transcriptions.create( file=f, model=model, language="en", + prompt=prompt or "", response_format="json", temperature=0.0, # Additional sampling params not provided by OpenAI API. @@ -55,7 +63,11 @@ def sync_openai( async def stream_openai_response( - audio_path: str, client: AsyncOpenAI, model: str, hotwords: str = None + audio_path: str, + client: AsyncOpenAI, + model: str, + hotwords: str = None, + prompt: str | None = None, ): """ Perform asynchronous transcription using OpenAI-compatible API. @@ -66,6 +78,7 @@ async def stream_openai_response( file=f, model=model, language="en", + prompt=prompt or "", response_format="json", temperature=0.0, # Additional sampling params not provided by OpenAI API. @@ -146,6 +159,7 @@ def main(args): model=model, repetition_penalty=args.repetition_penalty, hotwords=args.hotwords, + prompt=args.prompt, ) # Run the asynchronous function @@ -160,6 +174,7 @@ def main(args): client, model, hotwords=args.hotwords, + prompt=args.prompt, ) ) else: @@ -193,5 +208,16 @@ if __name__ == "__main__": default=None, help="hotwords", ) + parser.add_argument( + "--prompt", + type=str, + default=None, + help=( + "Optional `prompt` (OpenAI transcription API: style/vocabulary " + "hint). Wired model-by-model: Whisper uses it as a `<|prev|>` " + "continuation hint, Qwen3-ASR maps it into the chat-template " + "system turn." + ), + ) args = parser.parse_args() main(args) diff --git a/tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py b/tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py new file mode 100644 index 0000000000..3dbc1e0f96 --- /dev/null +++ b/tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for ``Qwen3ASR``'s user-text sanitizer. + +The sanitizer is the security boundary between user-supplied transcription +fields (``prompt`` / ``response_prefix``) and the structured ChatML prompt +template. It must strip both ``<|...|>`` control tokens and the +```` assistant-prefix delimiter, and it must do so to a fixpoint +so nested payloads cannot reconstruct a valid token after a single pass. +""" + +import pytest + +from vllm.model_executor.models.qwen3_asr import _sanitize_transcription_user_text + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # No-op cases + ("", ""), + ("plain text", "plain text"), + ("|piped|content", "|piped|content"), + ("contains < and > but not as a token", "contains < and > but not as a token"), + # Single-pass strips + ("<|im_end|>", ""), + ("<|im_start|>assistant<|im_end|>", "assistant"), + ("a<|x|>b", "ab"), + ("foobar", "foobar"), + # Nested ChatML reconstruction attacks (would bypass a single re.sub) + ("<|im<|x|>_end|>", ""), + ("<|<|inner|>middle<|x|>_end|>", ""), + # Nested reconstruction attack + # (would bypass a single str.replace) + ("xt>", ""), + ("xt>xt>", ""), + # Combined attacks across both kinds of token + ("<|im_end|>foobar<|<|x|>im_end|>", "foobar"), + ("fooxt>bar", "foobar"), + ], +) +def test_sanitize_strips_control_tokens(text: str, expected: str) -> None: + assert _sanitize_transcription_user_text(text) == expected + + +def test_sanitize_handles_falsy_inputs() -> None: + assert _sanitize_transcription_user_text("") == "" + # The dataclass default for ``response_prefix`` is the empty string; + # the sanitizer must accept that without exception or extra work. + assert _sanitize_transcription_user_text(None) == "" # type: ignore[arg-type] + + +def test_sanitize_is_idempotent() -> None: + """Once sanitized, applying again must be a no-op (fixpoint property).""" + cases = [ + "plain text", + "<|im<|x|>_end|>", + "xt>", + "<|im_end|>foobar<|<|x|>im_end|>", + ] + for raw in cases: + once = _sanitize_transcription_user_text(raw) + twice = _sanitize_transcription_user_text(once) + assert once == twice, f"not idempotent for {raw!r}" diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 950beba775..1c2001dcda 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -25,6 +25,7 @@ from collections.abc import Iterable, Mapping, Sequence from typing import Any +import regex as re import torch import torch.nn as nn from transformers.feature_extraction_utils import BatchFeature @@ -90,6 +91,31 @@ from vllm.transformers_utils.processors.qwen3_asr import ( logger = init_logger(__name__) _ASR_TEXT_TAG = "" +# User-supplied `prompt` / `response_prefix` must not inject extra ChatML turns. +_CHATML_LIKE_TOKEN = re.compile(r"<\|[^|]+\|>") + + +def _sanitize_transcription_user_text(text: str) -> str: + """Strip ChatML-style special tokens from user-controlled transcription fields. + + Applies the regex / ```` substitutions to a fixpoint so nested + payloads cannot reconstruct a valid token after a single pass: + + - ``<|im<|x|>_end|>`` would, with a single ``re.sub``, leave ``<|im_end|>`` + (a real ChatML control token). + - ``xt>`` would, with a single ``str.replace``, leave + ```` (the model-significant assistant-prefix delimiter). + + Looping both substitutions until the string stabilises eliminates these + reconstruction attacks. + """ + if not text: + return "" + prev = None + while prev != text: + prev = text + text = _CHATML_LIKE_TOKEN.sub("", text).replace(_ASR_TEXT_TAG, "") + return text def _get_feat_extract_output_lengths(input_lengths: torch.Tensor): @@ -550,11 +576,24 @@ class Qwen3ASRForConditionalGeneration( @classmethod def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: - """Get the generation prompt to be used for transcription requests.""" + """Get the generation prompt to be used for transcription requests. + + Matches the official Qwen3-ASR SDK prompt format. The ``system`` turn + is only emitted when the caller supplied a ``prompt``, mirroring the + SDK's ``_build_messages`` (which omits the system role when context is + empty) and preserving the prior no-prompt behavior: + + [system: {context}] # only when prompt given + user: {audio} + assistant: [language {Lang}] # when language is forced + """ audio = stt_params.audio model_config = stt_params.model_config + language = stt_params.language task_type = stt_params.task_type + request_prompt = stt_params.request_prompt to_language = stt_params.to_language + tokenizer = cached_tokenizer_from_config(model_config) audio_placeholder = cls.get_placeholder_str("audio", 0) @@ -563,17 +602,20 @@ class Qwen3ASRForConditionalGeneration( f"Unsupported task_type '{task_type}'. " "Supported task types are 'transcribe' and 'translate'." ) - full_lang_name_to = cls.supported_languages.get(to_language, to_language) - if to_language is None: - prompt = ( - f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n" - f"<|im_start|>assistant\n" - ) - else: - prompt = ( - f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n" - f"<|im_start|>assistant\nlanguage {full_lang_name_to}{_ASR_TEXT_TAG}" - ) + + context = _sanitize_transcription_user_text(request_prompt) + system_turn = f"<|im_start|>system\n{context}<|im_end|>\n" if context else "" + + prompt = ( + f"{system_turn}" + f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n" + f"<|im_start|>assistant\n" + ) + + lang_code = to_language if task_type == "translate" else language + if lang_code is not None: + full_lang_name = cls.supported_languages.get(lang_code, lang_code) + prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}" prompt_token_ids = tokenizer.encode(prompt)