[Audio] Improve Audio Inference Scripts (offline/online) (#29279)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2025-12-31 23:34:18 +00:00
committed by GitHub
parent 21de6d4b02
commit adcf682fc7
2 changed files with 113 additions and 32 deletions
+31 -18
View File
@@ -495,27 +495,40 @@ def main(args):
temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
) )
mm_data = req_data.multi_modal_data def get_input(start, end):
if not mm_data: mm_data = req_data.multi_modal_data
mm_data = {} if not mm_data:
if audio_count > 0: mm_data = {}
mm_data = { if end - start > 0:
"audio": [ mm_data = {
asset.audio_and_sample_rate for asset in audio_assets[:audio_count] "audio": [
] asset.audio_and_sample_rate for asset in audio_assets[start:end]
} ]
}
inputs = {"multi_modal_data": mm_data}
if req_data.prompt:
inputs["prompt"] = req_data.prompt
else:
inputs["prompt_token_ids"] = req_data.prompt_token_ids
return inputs
# Batch inference
assert args.num_prompts > 0 assert args.num_prompts > 0
inputs = {"multi_modal_data": mm_data} if audio_count != 1:
inputs = get_input(0, audio_count)
if req_data.prompt:
inputs["prompt"] = req_data.prompt
else:
inputs["prompt_token_ids"] = req_data.prompt_token_ids
if args.num_prompts > 1:
# Batch inference
inputs = [inputs] * args.num_prompts inputs = [inputs] * args.num_prompts
else:
# For single audio input, we need to vary the audio input
# to avoid deduplication in vLLM engine.
inputs = []
for i in range(args.num_prompts):
start = i % len(audio_assets)
inp = get_input(start, start + 1)
inputs.append(inp)
# Add LoRA request if applicable # Add LoRA request if applicable
lora_request = ( lora_request = (
req_data.lora_requests * args.num_prompts if req_data.lora_requests else None req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
@@ -18,6 +18,7 @@ The script performs:
2. Streaming transcription using raw HTTP request to the vLLM server. 2. Streaming transcription using raw HTTP request to the vLLM server.
""" """
import argparse
import asyncio import asyncio
from openai import AsyncOpenAI, OpenAI from openai import AsyncOpenAI, OpenAI
@@ -25,14 +26,14 @@ from openai import AsyncOpenAI, OpenAI
from vllm.assets.audio import AudioAsset from vllm.assets.audio import AudioAsset
def sync_openai(audio_path: str, client: OpenAI): def sync_openai(audio_path: str, client: OpenAI, model: str):
""" """
Perform synchronous transcription using OpenAI-compatible API. Perform synchronous transcription using OpenAI-compatible API.
""" """
with open(audio_path, "rb") as f: with open(audio_path, "rb") as f:
transcription = client.audio.transcriptions.create( transcription = client.audio.transcriptions.create(
file=f, file=f,
model="openai/whisper-large-v3", model=model,
language="en", language="en",
response_format="json", response_format="json",
temperature=0.0, temperature=0.0,
@@ -42,18 +43,18 @@ def sync_openai(audio_path: str, client: OpenAI):
repetition_penalty=1.3, repetition_penalty=1.3,
), ),
) )
print("transcription result:", transcription.text) print("transcription result [sync]:", transcription.text)
async def stream_openai_response(audio_path: str, client: AsyncOpenAI): async def stream_openai_response(audio_path: str, client: AsyncOpenAI, model: str):
""" """
Perform asynchronous transcription using OpenAI-compatible API. Perform asynchronous transcription using OpenAI-compatible API.
""" """
print("\ntranscription result:", end=" ") print("\ntranscription result [stream]:", end=" ")
with open(audio_path, "rb") as f: with open(audio_path, "rb") as f:
transcription = await client.audio.transcriptions.create( transcription = await client.audio.transcriptions.create(
file=f, file=f,
model="openai/whisper-large-v3", model=model,
language="en", language="en",
response_format="json", response_format="json",
temperature=0.0, temperature=0.0,
@@ -72,7 +73,47 @@ async def stream_openai_response(audio_path: str, client: AsyncOpenAI):
print() # Final newline after stream ends print() # Final newline after stream ends
def main(): def stream_api_response(audio_path: str, model: str, openai_api_base: str):
"""
Perform streaming transcription using raw HTTP requests to the vLLM API server.
"""
import json
import os
import requests
api_url = f"{openai_api_base}/audio/transcriptions"
headers = {"User-Agent": "Transcription-Client"}
with open(audio_path, "rb") as f:
files = {"file": (os.path.basename(audio_path), f)}
data = {
"stream": "true",
"model": model,
"language": "en",
"response_format": "json",
}
print("\ntranscription result [stream]:", end=" ")
response = requests.post(
api_url, headers=headers, files=files, data=data, stream=True
)
for chunk in response.iter_lines(
chunk_size=8192, decode_unicode=False, delimiter=b"\n"
):
if chunk:
data = chunk[len("data: ") :]
data = json.loads(data.decode("utf-8"))
data = data["choices"][0]
delta = data["delta"]["content"]
print(delta, end="", flush=True)
finish_reason = data.get("finish_reason")
if finish_reason is not None:
print(f"\n[Stream finished reason: {finish_reason}]")
break
def main(args):
mary_had_lamb = str(AudioAsset("mary_had_lamb").get_local_path()) mary_had_lamb = str(AudioAsset("mary_had_lamb").get_local_path())
winning_call = str(AudioAsset("winning_call").get_local_path()) winning_call = str(AudioAsset("winning_call").get_local_path())
@@ -84,14 +125,41 @@ def main():
base_url=openai_api_base, base_url=openai_api_base,
) )
sync_openai(mary_had_lamb, client) model = client.models.list().data[0].id
print(f"Using model: {model}")
# Run the synchronous function
sync_openai(args.audio_path if args.audio_path else mary_had_lamb, client, model)
# Run the asynchronous function # Run the asynchronous function
client = AsyncOpenAI( if "openai" in model:
api_key=openai_api_key, client = AsyncOpenAI(
base_url=openai_api_base, api_key=openai_api_key,
) base_url=openai_api_base,
asyncio.run(stream_openai_response(winning_call, client)) )
asyncio.run(
stream_openai_response(
args.audio_path if args.audio_path else winning_call, client, model
)
)
else:
stream_api_response(
args.audio_path if args.audio_path else winning_call,
model,
openai_api_base,
)
if __name__ == "__main__": if __name__ == "__main__":
main() # setup argparser
parser = argparse.ArgumentParser(
description="OpenAI Transcription Client using vLLM API Server"
)
parser.add_argument(
"--audio_path",
type=str,
default=None,
help="The path to the audio file to transcribe.",
)
args = parser.parse_args()
main(args)