[Examples] Resettle generate examples. (#36464)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,194 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Examples of batched chat completions via the vLLM OpenAI-compatible API.
|
||||
|
||||
The /v1/chat/completions/batch endpoint accepts ``messages`` as a list of
|
||||
conversations. Each conversation is processed independently and the response
|
||||
contains one choice per conversation, indexed 0, 1, ..., N-1.
|
||||
|
||||
Start a server first, e.g.:
|
||||
vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000
|
||||
|
||||
Current limitations compared to /v1/chat/completions:
|
||||
- Streaming is not supported.
|
||||
- Tool use is not supported.
|
||||
- Beam search is not supported.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
BASE_URL = os.environ.get("VLLM_BASE_URL", "http://localhost:8000")
|
||||
MODEL = os.environ.get("VLLM_MODEL", "Qwen/Qwen2.5-1.5B-Instruct")
|
||||
BATCH_URL = f"{BASE_URL}/v1/chat/completions/batch"
|
||||
|
||||
|
||||
def post_batch(payload: dict) -> dict:
|
||||
response = httpx.post(BATCH_URL, json=payload, timeout=60)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print("=== Example 1a: single conversation (standard endpoint) ===")
|
||||
response = httpx.post(
|
||||
f"{BASE_URL}/v1/chat/completions",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "What is the capital of Japan?"}],
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
for choice in data["choices"]:
|
||||
print(f" [{choice['index']}] {choice['message']['content']}")
|
||||
|
||||
print("\n=== Example 1b: batched plain text (2 conversations) ===")
|
||||
data = post_batch(
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
[{"role": "user", "content": "What is the capital of France?"}],
|
||||
[{"role": "user", "content": "What is the capital of Japan?"}],
|
||||
],
|
||||
}
|
||||
)
|
||||
for choice in data["choices"]:
|
||||
print(f" [{choice['index']}] {choice['message']['content']}")
|
||||
|
||||
print("\n=== Example 2: batch with regex constraint (yes|no) ===")
|
||||
data = post_batch(
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
[{"role": "user", "content": "Is the sky blue? Answer yes or no."}],
|
||||
[{"role": "user", "content": "Is fire cold? Answer yes or no."}],
|
||||
],
|
||||
"structured_outputs": {"regex": "(yes|no)"},
|
||||
}
|
||||
)
|
||||
for choice in data["choices"]:
|
||||
print(f" [{choice['index']}] {choice['message']['content']}")
|
||||
|
||||
print("\n=== Example 3: batch with json_schema ===")
|
||||
person_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string", "description": "Full name of the person"},
|
||||
"age": {"type": "integer", "description": "Age in years"},
|
||||
},
|
||||
"required": ["name", "age"],
|
||||
}
|
||||
data = post_batch(
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Describe the person: name Alice, age 30.",
|
||||
}
|
||||
],
|
||||
[{"role": "user", "content": "Describe the person: name Bob, age 25."}],
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "person",
|
||||
"strict": True,
|
||||
"schema": person_schema,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
for choice in data["choices"]:
|
||||
person = json.loads(choice["message"]["content"])
|
||||
print(f" [{choice['index']}] {person}")
|
||||
|
||||
print("\n=== Example 4: batch book summaries ===")
|
||||
book_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"author": {
|
||||
"type": "string",
|
||||
"description": "Full name of the author",
|
||||
},
|
||||
"num_pages": {
|
||||
"type": "integer",
|
||||
"description": "Number of pages in the book",
|
||||
},
|
||||
"short_summary": {
|
||||
"type": "string",
|
||||
"description": "A one-sentence summary of the book",
|
||||
},
|
||||
"long_summary": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"A detailed two to three sentence summary covering "
|
||||
"the main themes and plot"
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["author", "num_pages", "short_summary", "long_summary"],
|
||||
}
|
||||
system_msg = {
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a literary analyst. Extract structured information "
|
||||
"from book descriptions."
|
||||
),
|
||||
}
|
||||
data = post_batch(
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
[
|
||||
system_msg,
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Extract information from this book: '1984' by George"
|
||||
" Orwell, published in 1949, 328 pages. A dystopian"
|
||||
" novel set in a totalitarian society ruled by Big"
|
||||
" Brother, following Winston Smith as he secretly"
|
||||
" rebels against the oppressive Party that surveils"
|
||||
" and controls every aspect of life."
|
||||
),
|
||||
},
|
||||
],
|
||||
[
|
||||
system_msg,
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Extract information from this book: 'The Hitchhiker's"
|
||||
" Guide to the Galaxy' by Douglas Adams, published in"
|
||||
" 1979, 193 pages. A comedic science fiction novel"
|
||||
" following Arthur Dent, an ordinary Englishman who is"
|
||||
" whisked off Earth moments before it is demolished to"
|
||||
" make way for a hyperspace bypass, and his subsequent"
|
||||
" absurd adventures across the universe."
|
||||
),
|
||||
},
|
||||
],
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "book_summary",
|
||||
"strict": True,
|
||||
"schema": book_schema,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
for choice in data["choices"]:
|
||||
book = json.loads(choice["message"]["content"])
|
||||
print(f" [{choice['index']}] {book}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,416 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""An example showing how to use vLLM to serve multimodal models
|
||||
and run online serving with OpenAI client.
|
||||
|
||||
Launch the vLLM server with the following command:
|
||||
|
||||
(single image inference with Llava)
|
||||
vllm serve llava-hf/llava-1.5-7b-hf
|
||||
|
||||
(multi-image inference with Phi-3.5-vision-instruct)
|
||||
vllm serve microsoft/Phi-3.5-vision-instruct --runner generate \
|
||||
--trust-remote-code --max-model-len 4096 --limit-mm-per-prompt.image 2
|
||||
|
||||
(audio inference with Ultravox)
|
||||
vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b \
|
||||
--max-model-len 4096 --trust-remote-code
|
||||
|
||||
run the script with
|
||||
python openai_chat_completion_client_for_multimodal.py --chat-type audio
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pybase64 as base64
|
||||
import requests
|
||||
from openai import OpenAI
|
||||
from utils import get_first_model
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
client = OpenAI(
|
||||
# defaults to os.environ.get("OPENAI_API_KEY")
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
headers = {"User-Agent": "vLLM Example Client"}
|
||||
|
||||
|
||||
def encode_base64_content_from_url(content_url: str) -> str:
|
||||
"""Encode a content retrieved from a remote url to base64 format."""
|
||||
|
||||
with requests.get(content_url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
result = base64.b64encode(response.content).decode("utf-8")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def encode_base64_content_from_file(file_path: str) -> str:
|
||||
"""Encode a local file content to base64 format."""
|
||||
|
||||
with open(file_path, "rb") as file:
|
||||
file_content = file.read()
|
||||
result = base64.b64encode(file_content).decode("utf-8")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# Text-only inference
|
||||
def run_text_only(model: str, max_completion_tokens: int) -> None:
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=[{"role": "user", "content": "What's the capital of France?"}],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion.choices[0].message.content
|
||||
print("Chat completion output:\n", result)
|
||||
|
||||
|
||||
# Single-image input inference
|
||||
def run_single_image(model: str, max_completion_tokens: int) -> None:
|
||||
## Use image url in the payload
|
||||
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||
image_file = "/path/to/image.jpg" # local file
|
||||
chat_completion_from_url = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_url.choices[0].message.content
|
||||
print("Chat completion output from image url:\n", result)
|
||||
|
||||
## Use local image url in the payload
|
||||
# Launch the API server/engine with the --allowed-local-media-path argument.
|
||||
if os.path.exists(image_file):
|
||||
chat_completion_from_local_image_url = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"file://{image_file}"},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
result = chat_completion_from_local_image_url.choices[0].message.content
|
||||
print("Chat completion output from local image file:\n", result)
|
||||
else:
|
||||
print(f"Local image file not found at {image_file}, skipping local file test.")
|
||||
|
||||
## Use base64 encoded image in the payload
|
||||
image_base64 = encode_base64_content_from_url(image_url)
|
||||
chat_completion_from_base64 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{image_base64}"},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_base64.choices[0].message.content
|
||||
print("Chat completion output from base64 encoded image:", result)
|
||||
|
||||
## Use base64 encoded local image in the payload
|
||||
if os.path.exists(image_file):
|
||||
local_image_base64 = encode_base64_content_from_file(image_file)
|
||||
chat_completion_from_local_image_base64 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/jpeg;base64,{local_image_base64}"
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_local_image_base64.choices[0].message.content
|
||||
print("Chat completion output from base64 encoded local image:", result)
|
||||
else:
|
||||
print(f"Local image file not found at {image_file}, skipping local file test.")
|
||||
|
||||
|
||||
# Multi-image input inference
|
||||
def run_multi_image(model: str, max_completion_tokens: int) -> None:
|
||||
image_url_duck = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/duck.jpg"
|
||||
image_url_lion = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/lion.jpg"
|
||||
chat_completion_from_url = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What are the animals in these images?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url_duck},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url_lion},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_url.choices[0].message.content
|
||||
print("Chat completion output:\n", result)
|
||||
|
||||
|
||||
# Video input inference
|
||||
def run_video(model: str, max_completion_tokens: int) -> None:
|
||||
video_url = "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
|
||||
video_base64 = encode_base64_content_from_url(video_url)
|
||||
|
||||
## Use video url in the payload
|
||||
chat_completion_from_url = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this video?"},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_url.choices[0].message.content
|
||||
print("Chat completion output from video url:\n", result)
|
||||
|
||||
## Use base64 encoded video in the payload
|
||||
chat_completion_from_base64 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this video?"},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_base64.choices[0].message.content
|
||||
print("Chat completion output from base64 encoded video:\n", result)
|
||||
|
||||
|
||||
# Audio input inference
|
||||
def run_audio(model: str, max_completion_tokens: int) -> None:
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
audio_url = AudioAsset("winning_call").url
|
||||
audio_base64 = encode_base64_content_from_url(audio_url)
|
||||
|
||||
# OpenAI-compatible schema (`input_audio`)
|
||||
chat_completion_from_base64 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this audio?"},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
# Any format supported by soundfile/PyAV is supported
|
||||
"data": audio_base64,
|
||||
"format": "wav",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_base64.choices[0].message.content
|
||||
print("Chat completion output from input audio:\n", result)
|
||||
|
||||
# HTTP URL
|
||||
chat_completion_from_url = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this audio?"},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
# Any format supported by soundfile/PyAV is supported
|
||||
"url": audio_url
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_url.choices[0].message.content
|
||||
print("Chat completion output from audio url:\n", result)
|
||||
|
||||
# base64 URL
|
||||
chat_completion_from_base64 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this audio?"},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
# Any format supported by soundfile/PyAV is supported
|
||||
"url": f"data:audio/ogg;base64,{audio_base64}"
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_base64.choices[0].message.content
|
||||
print("Chat completion output from base64 encoded audio:\n", result)
|
||||
|
||||
|
||||
def run_multi_audio(model: str, max_completion_tokens: int) -> None:
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
# Two different audios to showcase batched inference.
|
||||
audio_url = AudioAsset("winning_call").url
|
||||
audio_base64 = encode_base64_content_from_url(audio_url)
|
||||
audio_url2 = AudioAsset("azacinto_foscolo").url
|
||||
audio_base64_2 = encode_base64_content_from_url(audio_url2)
|
||||
|
||||
# OpenAI-compatible schema (`input_audio`)
|
||||
chat_completion_from_base64 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Are these two audios the same?"},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": audio_base64,
|
||||
"format": "wav",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": audio_base64_2,
|
||||
"format": "wav",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
)
|
||||
|
||||
result = chat_completion_from_base64.choices[0].message.content
|
||||
print("Chat completion output from input audio:\n", result)
|
||||
|
||||
|
||||
example_function_map = {
|
||||
"text-only": run_text_only,
|
||||
"single-image": run_single_image,
|
||||
"multi-image": run_multi_image,
|
||||
"multi-audio": run_multi_audio,
|
||||
"video": run_video,
|
||||
"audio": run_audio,
|
||||
}
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Demo on using OpenAI client for online serving with "
|
||||
"multimodal language models served with vLLM."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-type",
|
||||
"-c",
|
||||
type=str,
|
||||
default="single-image",
|
||||
choices=list(example_function_map.keys()),
|
||||
help="Conversation type with multimodal data.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-completion-tokens",
|
||||
"-n",
|
||||
type=int,
|
||||
default=128,
|
||||
help="Maximum number of tokens to generate for each completion.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main(args) -> None:
|
||||
chat_type = args.chat_type
|
||||
model = get_first_model(client)
|
||||
example_function_map[chat_type](model, args.max_completion_tokens)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
main(args)
|
||||
@@ -1,195 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled. For example:
|
||||
|
||||
IMPORTANT: for mistral, you must use one of the provided mistral tool call
|
||||
templates, or your own - the model default doesn't work for tool calls with vLLM
|
||||
See the vLLM docs on OpenAI server & tool calling for more details.
|
||||
|
||||
vllm serve mistralai/Mistral-7B-Instruct-v0.3 \
|
||||
--chat-template examples/tool_chat_template_mistral.jinja \
|
||||
--enable-auto-tool-choice --tool-call-parser mistral
|
||||
|
||||
OR
|
||||
vllm serve NousResearch/Hermes-2-Pro-Llama-3-8B \
|
||||
--chat-template examples/tool_chat_template_hermes.jinja \
|
||||
--enable-auto-tool-choice --tool-call-parser hermes
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
properties = {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "The city to find the weather for, e.g. 'San Francisco'",
|
||||
},
|
||||
"state": {
|
||||
"type": "string",
|
||||
"description": "the two-letter abbreviation for the state that the city is"
|
||||
" in, e.g. 'CA' which would mean 'California'",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"description": "The unit to fetch the temperature in",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
}
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": ["city", "state", "unit"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Hi! How are you doing today?"},
|
||||
{"role": "assistant", "content": "I'm doing well! How can I help you?"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Can you tell me what the temperate will be in Dallas, in fahrenheit?"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def get_current_weather(city: str, state: str, unit: "str"):
|
||||
return (
|
||||
"The weather in Dallas, Texas is 85 degrees fahrenheit. It is "
|
||||
"partly cloudly, with highs in the 90's."
|
||||
)
|
||||
|
||||
|
||||
def handle_tool_calls_stream(
|
||||
client: OpenAI,
|
||||
messages: list[dict[str, str]],
|
||||
model: str,
|
||||
tools: list[dict[str, Any]],
|
||||
) -> list[Any]:
|
||||
tool_calls_stream = client.chat.completions.create(
|
||||
messages=messages, model=model, tools=tools, stream=True
|
||||
)
|
||||
chunks = []
|
||||
print("chunks: ")
|
||||
for chunk in tool_calls_stream:
|
||||
chunks.append(chunk)
|
||||
if chunk.choices[0].delta.tool_calls:
|
||||
print(chunk.choices[0].delta.tool_calls[0])
|
||||
else:
|
||||
print(chunk.choices[0].delta)
|
||||
return chunks
|
||||
|
||||
|
||||
def handle_tool_calls_arguments(chunks: list[Any]) -> list[str]:
|
||||
arguments = []
|
||||
tool_call_idx = -1
|
||||
print("arguments: ")
|
||||
for chunk in chunks:
|
||||
if chunk.choices[0].delta.tool_calls:
|
||||
tool_call = chunk.choices[0].delta.tool_calls[0]
|
||||
if tool_call.index != tool_call_idx:
|
||||
if tool_call_idx >= 0:
|
||||
print(f"streamed tool call arguments: {arguments[tool_call_idx]}")
|
||||
tool_call_idx = chunk.choices[0].delta.tool_calls[0].index
|
||||
arguments.append("")
|
||||
if tool_call.id:
|
||||
print(f"streamed tool call id: {tool_call.id} ")
|
||||
|
||||
if tool_call.function:
|
||||
if tool_call.function.name:
|
||||
print(f"streamed tool call name: {tool_call.function.name}")
|
||||
|
||||
if tool_call.function.arguments:
|
||||
arguments[tool_call_idx] += tool_call.function.arguments
|
||||
|
||||
return arguments
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize OpenAI client
|
||||
client = OpenAI(
|
||||
# defaults to os.environ.get("OPENAI_API_KEY")
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
# Get available models and select one
|
||||
models = client.models.list()
|
||||
model = models.data[0].id
|
||||
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=messages, model=model, tools=tools
|
||||
)
|
||||
|
||||
print("-" * 70)
|
||||
print("Chat completion results:")
|
||||
print(chat_completion)
|
||||
print("-" * 70)
|
||||
|
||||
# Stream tool calls
|
||||
chunks = handle_tool_calls_stream(client, messages, model, tools)
|
||||
print("-" * 70)
|
||||
|
||||
# Handle arguments from streamed tool calls
|
||||
arguments = handle_tool_calls_arguments(chunks)
|
||||
|
||||
if len(arguments):
|
||||
print(f"streamed tool call arguments: {arguments[-1]}\n")
|
||||
|
||||
print("-" * 70)
|
||||
|
||||
# Add tool call results to the conversation
|
||||
messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": chat_completion.choices[0].message.tool_calls,
|
||||
"reasoning": chat_completion.choices[0].message.reasoning,
|
||||
}
|
||||
)
|
||||
|
||||
# Now, simulate a tool call
|
||||
available_tools = {"get_current_weather": get_current_weather}
|
||||
|
||||
completion_tool_calls = chat_completion.choices[0].message.tool_calls
|
||||
for call in completion_tool_calls:
|
||||
tool_to_call = available_tools[call.function.name]
|
||||
args = json.loads(call.function.arguments)
|
||||
result = tool_to_call(**args)
|
||||
print("tool_to_call result: ", result)
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"content": result,
|
||||
"tool_call_id": call.id,
|
||||
"name": call.function.name,
|
||||
}
|
||||
)
|
||||
|
||||
chat_completion_2 = client.chat.completions.create(
|
||||
messages=messages, model=model, tools=tools, stream=False
|
||||
)
|
||||
print("Chat completion2 results:")
|
||||
print(chat_completion_2)
|
||||
print("-" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,130 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
To run this example, you can start the vLLM server
|
||||
without any specific flags:
|
||||
|
||||
```bash
|
||||
vllm serve unsloth/Llama-3.2-1B-Instruct \
|
||||
--structured-outputs-config.backend outlines
|
||||
```
|
||||
|
||||
This example demonstrates how to generate chat completions
|
||||
using the OpenAI Python client library.
|
||||
"""
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "The city to find the weather for"
|
||||
", e.g. 'San Francisco'",
|
||||
},
|
||||
"state": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"the two-letter abbreviation for the state that the "
|
||||
"city is in, e.g. 'CA' which would mean 'California'"
|
||||
),
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"description": "The unit to fetch the temperature in",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["city", "state", "unit"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_forecast",
|
||||
"description": "Get the weather forecast for a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"The city to get the forecast for, e.g. 'New York'"
|
||||
),
|
||||
},
|
||||
"state": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"The two-letter abbreviation for the state, e.g. 'NY'"
|
||||
),
|
||||
},
|
||||
"days": {
|
||||
"type": "integer",
|
||||
"description": "Number of days to get the forecast for (1-7)",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"description": "The unit to fetch the temperature in",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["city", "state", "days", "unit"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Hi! How are you doing today?"},
|
||||
{"role": "assistant", "content": "I'm doing well! How can I help you?"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Can you tell me what the current weather is in Dallas \
|
||||
and the forecast for the next 5 days, in fahrenheit?",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
client = OpenAI(
|
||||
# defaults to os.environ.get("OPENAI_API_KEY")
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
models = client.models.list()
|
||||
model = models.data[0].id
|
||||
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=messages,
|
||||
model=model,
|
||||
tools=tools,
|
||||
tool_choice="required",
|
||||
stream=True, # Enable streaming response
|
||||
)
|
||||
|
||||
for chunk in chat_completion:
|
||||
if chunk.choices and chunk.choices[0].delta.tool_calls:
|
||||
print(chunk.choices[0].delta.tool_calls)
|
||||
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=messages, model=model, tools=tools, tool_choice="required"
|
||||
)
|
||||
|
||||
print(chat_completion.choices[0].message.tool_calls)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,245 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled for xLAM-2 models:
|
||||
|
||||
vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
|
||||
|
||||
OR
|
||||
|
||||
vllm serve --model Salesforce/xLAM-2-3b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "empty"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
|
||||
# Define tool functions
|
||||
def get_weather(location: str, unit: str):
|
||||
return f"Weather in {location} is 22 degrees {unit}."
|
||||
|
||||
|
||||
def calculate_expression(expression: str):
|
||||
try:
|
||||
result = eval(expression)
|
||||
return f"The result of {expression} is {result}"
|
||||
except Exception as e:
|
||||
return f"Could not calculate {expression}: {e}"
|
||||
|
||||
|
||||
def translate_text(text: str, target_language: str):
|
||||
return f"Translation of '{text}' to {target_language}: [translated content]"
|
||||
|
||||
|
||||
# Define tools
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "City and state, e.g., 'San Francisco, CA'",
|
||||
},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["location", "unit"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calculate_expression",
|
||||
"description": "Calculate a mathematical expression",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expression": {
|
||||
"type": "string",
|
||||
"description": "Mathematical expression to evaluate, needs to be a valid python expression",
|
||||
}
|
||||
},
|
||||
"required": ["expression"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "translate_text",
|
||||
"description": "Translate text to another language",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {"type": "string", "description": "Text to translate"},
|
||||
"target_language": {
|
||||
"type": "string",
|
||||
"description": "Target language for translation",
|
||||
},
|
||||
},
|
||||
"required": ["text", "target_language"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# Map of function names to implementations
|
||||
tool_functions = {
|
||||
"get_weather": get_weather,
|
||||
"calculate_expression": calculate_expression,
|
||||
"translate_text": translate_text,
|
||||
}
|
||||
|
||||
|
||||
def process_response(response, tool_functions, original_query):
|
||||
"""Process a non-streaming response with possible tool calls"""
|
||||
|
||||
print("\n--- Response Output ---")
|
||||
|
||||
# Check if the response has content
|
||||
if response.choices[0].message.content:
|
||||
print(f"Content: {response.choices[0].message.content}")
|
||||
|
||||
# Check if the response has tool calls
|
||||
if response.choices[0].message.tool_calls:
|
||||
print("--------------------------------")
|
||||
print(f"Tool calls: {response.choices[0].message.tool_calls}")
|
||||
print("--------------------------------")
|
||||
|
||||
# Collect all tool calls and results before making follow-up request
|
||||
tool_results = []
|
||||
assistant_message = {"role": "assistant"}
|
||||
|
||||
if response.choices[0].message.content:
|
||||
assistant_message["content"] = response.choices[0].message.content
|
||||
|
||||
assistant_tool_calls = []
|
||||
|
||||
# Process each tool call
|
||||
for tool_call in response.choices[0].message.tool_calls:
|
||||
function_name = tool_call.function.name
|
||||
function_args = tool_call.function.arguments
|
||||
function_id = tool_call.id
|
||||
|
||||
print(f"Function called: {function_name}")
|
||||
print(f"Arguments: {function_args}")
|
||||
print(f"Function ID: {function_id}")
|
||||
|
||||
# Execute the function
|
||||
try:
|
||||
# Parse the JSON arguments
|
||||
args = json.loads(function_args)
|
||||
|
||||
# Call the function with the arguments
|
||||
function_result = tool_functions[function_name](**args)
|
||||
print(f"\n--- Function Result ---\n{function_result}\n")
|
||||
|
||||
# Add tool call to assistant message
|
||||
assistant_tool_calls.append(
|
||||
{
|
||||
"id": function_id,
|
||||
"type": "function",
|
||||
"function": {"name": function_name, "arguments": function_args},
|
||||
}
|
||||
)
|
||||
|
||||
# Add tool result to tool_results
|
||||
tool_results.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": function_id,
|
||||
"content": function_result,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error executing function: {e}")
|
||||
|
||||
# Add tool_calls to assistant message
|
||||
assistant_message["tool_calls"] = assistant_tool_calls
|
||||
|
||||
# Create a follow-up message with all function results
|
||||
follow_up_messages = [
|
||||
{"role": "user", "content": original_query},
|
||||
assistant_message,
|
||||
]
|
||||
|
||||
# Add all tool results to the messages
|
||||
follow_up_messages.extend(tool_results)
|
||||
|
||||
# Get completion with all tool results in a single follow-up
|
||||
follow_up_response = client.chat.completions.create(
|
||||
model=client.models.list().data[0].id,
|
||||
messages=follow_up_messages,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
print("\n--- Follow-up Response ---")
|
||||
print(follow_up_response.choices[0].message.content)
|
||||
print("--- End Follow-up ---\n")
|
||||
|
||||
print("--- End Response ---\n")
|
||||
|
||||
|
||||
def run_test_case(query, test_name):
|
||||
"""Run a single test case with the given query"""
|
||||
print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}")
|
||||
print(f"Query: '{query}'")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Create non-streaming chat completion request
|
||||
response = client.chat.completions.create(
|
||||
model=client.models.list().data[0].id,
|
||||
messages=[{"role": "user", "content": query}],
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
stream=False,
|
||||
)
|
||||
|
||||
# Process the non-streaming response, passing the original query
|
||||
process_response(response, tool_functions, query)
|
||||
|
||||
end_time = time.time()
|
||||
print(f"Test completed in {end_time - start_time:.2f} seconds")
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize OpenAI client
|
||||
global client
|
||||
client = OpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
# Run test cases
|
||||
test_cases = [
|
||||
("I want to know the weather in San Francisco", "Weather Information"),
|
||||
("Calculate 25 * 17 + 31", "Math Calculation"),
|
||||
("Translate 'Hello world' to Spanish", "Text Translation"),
|
||||
("What is the weather in Tokyo and New York in celsius", "Multiple Tool Usage"),
|
||||
]
|
||||
|
||||
# Execute all test cases
|
||||
for query, test_name in test_cases:
|
||||
run_test_case(query, test_name)
|
||||
time.sleep(1) # Small delay between tests
|
||||
|
||||
print("\nAll tests completed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,273 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled for xLAM-2 models:
|
||||
|
||||
vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
|
||||
|
||||
OR
|
||||
|
||||
vllm serve --model Salesforce/xLAM-2-3b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
|
||||
|
||||
This example demonstrates streaming tool calls with xLAM models.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "empty"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
|
||||
# Define tool functions
|
||||
def get_weather(location: str, unit: str):
|
||||
return f"Weather in {location} is 22 degrees {unit}."
|
||||
|
||||
|
||||
def calculate_expression(expression: str):
|
||||
try:
|
||||
result = eval(expression)
|
||||
return f"The result of {expression} is {result}"
|
||||
except Exception as e:
|
||||
return f"Could not calculate {expression}: {e}"
|
||||
|
||||
|
||||
def translate_text(text: str, target_language: str):
|
||||
return f"Translation of '{text}' to {target_language}: [translated content]"
|
||||
|
||||
|
||||
# Define tools
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "City and state, e.g., 'San Francisco, CA'",
|
||||
},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["location", "unit"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calculate_expression",
|
||||
"description": "Calculate a mathematical expression",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expression": {
|
||||
"type": "string",
|
||||
"description": "Mathematical expression to evaluate, needs to be a valid Python expression",
|
||||
}
|
||||
},
|
||||
"required": ["expression"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "translate_text",
|
||||
"description": "Translate text to another language",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {"type": "string", "description": "Text to translate"},
|
||||
"target_language": {
|
||||
"type": "string",
|
||||
"description": "Target language for translation",
|
||||
},
|
||||
},
|
||||
"required": ["text", "target_language"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# Map of function names to implementations
|
||||
tool_functions = {
|
||||
"get_weather": get_weather,
|
||||
"calculate_expression": calculate_expression,
|
||||
"translate_text": translate_text,
|
||||
}
|
||||
|
||||
|
||||
def process_stream(response, tool_functions, original_query):
|
||||
"""Process a streaming response with possible tool calls"""
|
||||
# Track multiple tool calls
|
||||
tool_calls = {} # Dictionary to store tool calls by ID
|
||||
|
||||
current_id = None
|
||||
|
||||
print("\n--- Stream Output ---")
|
||||
for chunk in response:
|
||||
# Handle tool calls in the stream
|
||||
if chunk.choices[0].delta.tool_calls:
|
||||
for tool_call_chunk in chunk.choices[0].delta.tool_calls:
|
||||
# Get the tool call ID
|
||||
if hasattr(tool_call_chunk, "id") and tool_call_chunk.id:
|
||||
current_id = tool_call_chunk.id
|
||||
if current_id not in tool_calls:
|
||||
tool_calls[current_id] = {
|
||||
"function_name": None,
|
||||
"function_args": "",
|
||||
"function_id": current_id,
|
||||
}
|
||||
|
||||
# Extract function information as it comes in chunks
|
||||
if (
|
||||
hasattr(tool_call_chunk, "function")
|
||||
and current_id
|
||||
and current_id in tool_calls
|
||||
):
|
||||
if (
|
||||
hasattr(tool_call_chunk.function, "name")
|
||||
and tool_call_chunk.function.name
|
||||
):
|
||||
tool_calls[current_id]["function_name"] = (
|
||||
tool_call_chunk.function.name
|
||||
)
|
||||
print(f"Function called: {tool_call_chunk.function.name}")
|
||||
|
||||
if (
|
||||
hasattr(tool_call_chunk.function, "arguments")
|
||||
and tool_call_chunk.function.arguments
|
||||
):
|
||||
tool_calls[current_id]["function_args"] += (
|
||||
tool_call_chunk.function.arguments
|
||||
)
|
||||
print(f"Arguments chunk: {tool_call_chunk.function.arguments}")
|
||||
|
||||
# Handle regular content in the stream
|
||||
elif chunk.choices[0].delta.content:
|
||||
print(chunk.choices[0].delta.content, end="")
|
||||
|
||||
print("\n--- End Stream ---\n")
|
||||
|
||||
# Execute each function call and build messages for follow-up
|
||||
follow_up_messages = [{"role": "user", "content": original_query}]
|
||||
|
||||
for tool_id, tool_data in tool_calls.items():
|
||||
function_name = tool_data["function_name"]
|
||||
function_args = tool_data["function_args"]
|
||||
function_id = tool_data["function_id"]
|
||||
|
||||
if function_name and function_args:
|
||||
try:
|
||||
# Parse the JSON arguments
|
||||
args = json.loads(function_args)
|
||||
|
||||
# Call the function with the arguments
|
||||
function_result = tool_functions[function_name](**args)
|
||||
print(
|
||||
f"\n--- Function Result ({function_name}) ---\n{function_result}\n"
|
||||
)
|
||||
|
||||
# Add the assistant message with tool call
|
||||
follow_up_messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": function_id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": function_name,
|
||||
"arguments": function_args,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
# Add the tool message with function result
|
||||
follow_up_messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": function_id,
|
||||
"content": function_result,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error executing function: {e}")
|
||||
|
||||
# Only send follow-up if we have results to process
|
||||
if len(follow_up_messages) > 1:
|
||||
# Create a follow-up message with all the function results
|
||||
follow_up_response = client.chat.completions.create(
|
||||
model=client.models.list().data[0].id,
|
||||
messages=follow_up_messages,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
print("\n--- Follow-up Response ---")
|
||||
for chunk in follow_up_response:
|
||||
if chunk.choices[0].delta.content:
|
||||
print(chunk.choices[0].delta.content, end="")
|
||||
print("\n--- End Follow-up ---\n")
|
||||
|
||||
|
||||
def run_test_case(query, test_name):
|
||||
"""Run a single test case with the given query"""
|
||||
print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}")
|
||||
print(f"Query: '{query}'")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Create streaming chat completion request
|
||||
response = client.chat.completions.create(
|
||||
model=client.models.list().data[0].id,
|
||||
messages=[{"role": "user", "content": query}],
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
stream=True,
|
||||
)
|
||||
|
||||
# Process the streaming response
|
||||
process_stream(response, tool_functions, query)
|
||||
|
||||
end_time = time.time()
|
||||
print(f"Test completed in {end_time - start_time:.2f} seconds")
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize OpenAI client
|
||||
global client
|
||||
client = OpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
# Run test cases
|
||||
test_cases = [
|
||||
("I want to know the weather in San Francisco", "Weather Information"),
|
||||
("Calculate 25 * 17 + 31", "Math Calculation"),
|
||||
("Translate 'Hello world' to Spanish", "Text Translation"),
|
||||
("What is the weather in Tokyo and New York in celsius", "Multiple Tool Usage"),
|
||||
]
|
||||
|
||||
# Execute all test cases
|
||||
for query, test_name in test_cases:
|
||||
run_test_case(query, test_name)
|
||||
time.sleep(1) # Small delay between tests
|
||||
|
||||
print("\nAll tests completed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,170 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
An example demonstrates how to use tool calling with reasoning models
|
||||
like QwQ-32B. The reasoning will not be parsed by the tool
|
||||
calling process; only the final output will be parsed.
|
||||
|
||||
To run this example, you need to start the vLLM server with both
|
||||
the reasoning parser and tool calling enabled.
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/QwQ-32B \
|
||||
--reasoning-parser deepseek_r1 \
|
||||
--enable-auto-tool-choice --tool-call-parser hermes
|
||||
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
|
||||
# Now, simulate a tool call
|
||||
def get_current_weather(city: str, state: str, unit: "str"):
|
||||
return (
|
||||
"The weather in Dallas, Texas is 85 degrees fahrenheit. It is "
|
||||
"partly cloudly, with highs in the 90's."
|
||||
)
|
||||
|
||||
|
||||
available_tools = {"get_current_weather": get_current_weather}
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
properties = {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "The city to find the weather for, e.g. 'San Francisco'",
|
||||
},
|
||||
"state": {
|
||||
"type": "string",
|
||||
"description": "the two-letter abbreviation for the state that the city is"
|
||||
" in, e.g. 'CA' which would mean 'California'",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"description": "The unit to fetch the temperature in",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
}
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": ["city", "state", "unit"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
messages = [
|
||||
{"role": "user", "content": "Hi! How are you doing today?"},
|
||||
{"role": "assistant", "content": "I'm doing well! How can I help you?"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Can you tell me what the temperate will be in Dallas, in fahrenheit?"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def extract_reasoning_and_calls(chunks: list):
|
||||
reasoning = ""
|
||||
tool_call_idx = -1
|
||||
arguments = []
|
||||
function_names = []
|
||||
for chunk in chunks:
|
||||
if chunk.choices[0].delta.tool_calls:
|
||||
tool_call = chunk.choices[0].delta.tool_calls[0]
|
||||
if tool_call.index != tool_call_idx:
|
||||
tool_call_idx = chunk.choices[0].delta.tool_calls[0].index
|
||||
arguments.append("")
|
||||
function_names.append("")
|
||||
|
||||
if tool_call.function:
|
||||
if tool_call.function.name:
|
||||
function_names[tool_call_idx] = tool_call.function.name
|
||||
|
||||
if tool_call.function.arguments:
|
||||
arguments[tool_call_idx] += tool_call.function.arguments
|
||||
else:
|
||||
if hasattr(chunk.choices[0].delta, "reasoning"):
|
||||
reasoning += chunk.choices[0].delta.reasoning
|
||||
return reasoning, arguments, function_names
|
||||
|
||||
|
||||
def main():
|
||||
client = OpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
models = client.models.list()
|
||||
model = models.data[0].id
|
||||
|
||||
print("---------Full Generate With Automatic Function Calling-------------")
|
||||
tool_calls = client.chat.completions.create(
|
||||
messages=messages, model=model, tools=tools
|
||||
)
|
||||
print(f"reasoning: {tool_calls.choices[0].message.reasoning}")
|
||||
print(f"function name: {tool_calls.choices[0].message.tool_calls[0].function.name}")
|
||||
print(
|
||||
f"function arguments: "
|
||||
f"{tool_calls.choices[0].message.tool_calls[0].function.arguments}"
|
||||
)
|
||||
|
||||
print("----------Stream Generate With Automatic Function Calling-----------")
|
||||
tool_calls_stream = client.chat.completions.create(
|
||||
messages=messages, model=model, tools=tools, stream=True
|
||||
)
|
||||
|
||||
chunks = list(tool_calls_stream)
|
||||
|
||||
reasoning, arguments, function_names = extract_reasoning_and_calls(chunks)
|
||||
|
||||
print(f"reasoning: {reasoning}")
|
||||
print(f"function name: {function_names[0]}")
|
||||
print(f"function arguments: {arguments[0]}")
|
||||
|
||||
print("----------Full Generate With Named Function Calling-----------------")
|
||||
tool_calls = client.chat.completions.create(
|
||||
messages=messages,
|
||||
model=model,
|
||||
tools=tools,
|
||||
tool_choice={"type": "function", "function": {"name": "get_current_weather"}},
|
||||
)
|
||||
|
||||
tool_call = tool_calls.choices[0].message.tool_calls[0].function
|
||||
print(f"reasoning: {tool_calls.choices[0].message.reasoning}")
|
||||
print(f"function name: {tool_call.name}")
|
||||
print(f"function arguments: {tool_call.arguments}")
|
||||
print("----------Stream Generate With Named Function Calling--------------")
|
||||
|
||||
tool_calls_stream = client.chat.completions.create(
|
||||
messages=messages,
|
||||
model=model,
|
||||
tools=tools,
|
||||
tool_choice={"type": "function", "function": {"name": "get_current_weather"}},
|
||||
stream=True,
|
||||
)
|
||||
|
||||
chunks = list(tool_calls_stream)
|
||||
|
||||
reasoning, arguments, function_names = extract_reasoning_and_calls(chunks)
|
||||
print(f"reasoning: {reasoning}")
|
||||
print(f"function name: {function_names[0]}")
|
||||
print(f"function arguments: {arguments[0]}")
|
||||
print("\n\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
An example shows how to generate chat completions from reasoning models
|
||||
like DeepSeekR1.
|
||||
|
||||
To run this example, you need to start the vLLM server
|
||||
with the reasoning parser:
|
||||
|
||||
```bash
|
||||
vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \
|
||||
--reasoning-parser deepseek_r1
|
||||
```
|
||||
|
||||
This example demonstrates how to generate chat completions from reasoning models
|
||||
using the OpenAI Python client library.
|
||||
"""
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
|
||||
def main():
|
||||
client = OpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
models = client.models.list()
|
||||
model = models.data[0].id
|
||||
|
||||
# Round 1
|
||||
messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
|
||||
# ruff: noqa: E501
|
||||
# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
|
||||
response = client.chat.completions.create(model=model, messages=messages)
|
||||
|
||||
reasoning = response.choices[0].message.reasoning
|
||||
content = response.choices[0].message.content
|
||||
|
||||
print("reasoning for Round 1:", reasoning)
|
||||
print("content for Round 1:", content)
|
||||
|
||||
# Round 2
|
||||
messages.append({"role": "assistant", "content": content})
|
||||
messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": "How many Rs are there in the word 'strawberry'?",
|
||||
}
|
||||
)
|
||||
response = client.chat.completions.create(model=model, messages=messages)
|
||||
|
||||
reasoning = response.choices[0].message.reasoning
|
||||
content = response.choices[0].message.content
|
||||
|
||||
print("reasoning for Round 2:", reasoning)
|
||||
print("content for Round 2:", content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
An example shows how to generate chat completions from reasoning models
|
||||
like DeepSeekR1.
|
||||
|
||||
To run this example, you need to start the vLLM server with the reasoning
|
||||
parser:
|
||||
|
||||
```bash
|
||||
vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \
|
||||
--reasoning-parser deepseek_r1
|
||||
```
|
||||
|
||||
Unlike openai_chat_completion_with_reasoning.py, this example demonstrates the
|
||||
streaming chat completions feature.
|
||||
|
||||
The streaming chat completions feature allows you to receive chat completions
|
||||
in real-time as they are generated by the model. This is useful for scenarios
|
||||
where you want to display chat completions to the user as they are generated
|
||||
by the model.
|
||||
|
||||
Remember to check content and reasoning exist in `ChatCompletionChunk`,
|
||||
content may not exist leading to errors if you try to access it.
|
||||
"""
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
|
||||
messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
|
||||
|
||||
|
||||
def main():
|
||||
client = OpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
models = client.models.list()
|
||||
model = models.data[0].id
|
||||
|
||||
# ruff: noqa: E501
|
||||
# For granite: add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
|
||||
stream = client.chat.completions.create(model=model, messages=messages, stream=True)
|
||||
|
||||
print("client: Start streaming chat completions...")
|
||||
printed_reasoning = False
|
||||
printed_content = False
|
||||
|
||||
for chunk in stream:
|
||||
# Safely extract reasoning and content from delta,
|
||||
# defaulting to None if attributes don't exist or are empty strings
|
||||
reasoning = getattr(chunk.choices[0].delta, "reasoning", None) or None
|
||||
content = getattr(chunk.choices[0].delta, "content", None) or None
|
||||
|
||||
if reasoning is not None:
|
||||
if not printed_reasoning:
|
||||
printed_reasoning = True
|
||||
print("reasoning:", end="", flush=True)
|
||||
print(reasoning, end="", flush=True)
|
||||
elif content is not None:
|
||||
if not printed_content:
|
||||
printed_content = True
|
||||
print("\ncontent:", end="", flush=True)
|
||||
# Extract and print the content
|
||||
print(content, end="", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Language Identification (LID) demo using the FireRedLID model on vLLM.
|
||||
|
||||
FireRedLID is an audio encoder-decoder model that identifies the spoken
|
||||
language of an audio clip. Unlike ASR models that output full transcriptions,
|
||||
FireRedLID outputs at most 2 tokens representing the detected language
|
||||
(e.g. "en", "zh mandarin").
|
||||
|
||||
Start the vLLM server:
|
||||
|
||||
vllm serve PatchyTisa/FireRedLID-vllm
|
||||
|
||||
Then run this script:
|
||||
|
||||
# Use the built-in sample audio
|
||||
python examples/online_serving/openai_lid_client.py
|
||||
|
||||
# Use your own audio file(s)
|
||||
python examples/online_serving/openai_lid_client.py \
|
||||
--audio_paths audio_en.wav audio_zh.wav audio_fr.wav
|
||||
|
||||
# Batch-identify multiple files in one run
|
||||
python examples/online_serving/openai_lid_client.py \
|
||||
--audio_paths /path/to/dir/*.wav
|
||||
|
||||
Requirements:
|
||||
- vLLM with audio support
|
||||
- openai Python SDK
|
||||
- kaldi_native_fbank (pulled in by the model)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def identify_language(
|
||||
audio_path: str,
|
||||
client: OpenAI,
|
||||
model: str,
|
||||
) -> str:
|
||||
"""
|
||||
Send a single audio file to the vLLM transcription endpoint and return
|
||||
the detected language tag.
|
||||
|
||||
FireRedLID re-uses the OpenAI-compatible ``/v1/audio/transcriptions``
|
||||
endpoint. The "transcription" it returns is actually the language label
|
||||
(e.g. ``"en"`` or ``"zh mandarin"``).
|
||||
"""
|
||||
with open(audio_path, "rb") as f:
|
||||
result = client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=model,
|
||||
response_format="json",
|
||||
temperature=0.0,
|
||||
)
|
||||
return result.text.strip()
|
||||
|
||||
|
||||
def identify_language_raw(
|
||||
audio_path: str,
|
||||
model: str,
|
||||
api_base: str,
|
||||
) -> str:
|
||||
"""
|
||||
Same as :func:`identify_language` but uses raw HTTP so that the demo
|
||||
works without the ``openai`` SDK (useful for quick debugging).
|
||||
"""
|
||||
import requests
|
||||
|
||||
url = f"{api_base}/audio/transcriptions"
|
||||
with open(audio_path, "rb") as f:
|
||||
files = {"file": (os.path.basename(audio_path), f)}
|
||||
data = {
|
||||
"model": model,
|
||||
"response_format": "json",
|
||||
}
|
||||
resp = requests.post(url, files=files, data=data)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["text"].strip()
|
||||
|
||||
|
||||
def identify_language_streaming(
|
||||
audio_path: str,
|
||||
model: str,
|
||||
api_base: str,
|
||||
) -> str:
|
||||
"""
|
||||
Streaming variant – demonstrates the streaming transcription endpoint.
|
||||
For a 1-2 token output the stream finishes almost instantly, but this
|
||||
shows that the API path works end-to-end.
|
||||
"""
|
||||
import requests
|
||||
|
||||
url = f"{api_base}/audio/transcriptions"
|
||||
with open(audio_path, "rb") as f:
|
||||
files = {"file": (os.path.basename(audio_path), f)}
|
||||
data = {
|
||||
"stream": "true",
|
||||
"model": model,
|
||||
"response_format": "json",
|
||||
}
|
||||
response = requests.post(url, files=files, data=data, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
tokens: list[str] = []
|
||||
for chunk in response.iter_lines(
|
||||
chunk_size=8192, decode_unicode=False, delimiter=b"\n"
|
||||
):
|
||||
if not chunk:
|
||||
continue
|
||||
payload = json.loads(chunk[len("data: ") :].decode("utf-8"))
|
||||
choice = payload["choices"][0]
|
||||
delta = choice.get("delta", {}).get("content", "")
|
||||
if delta:
|
||||
tokens.append(delta)
|
||||
if choice.get("finish_reason") is not None:
|
||||
break
|
||||
|
||||
return "".join(tokens).strip()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Main
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
api_base = args.api_base.rstrip("/")
|
||||
client = OpenAI(api_key="EMPTY", base_url=api_base)
|
||||
model = client.models.list().data[0].id
|
||||
print(f"Model : {model}")
|
||||
print(f"Server: {api_base}\n")
|
||||
|
||||
# Resolve audio paths ------------------------------------------------
|
||||
if args.audio_paths:
|
||||
audio_paths = args.audio_paths
|
||||
else:
|
||||
# Fall back to the built-in vLLM sample audios (both are English).
|
||||
audio_paths = [
|
||||
str(AudioAsset("mary_had_lamb").get_local_path()),
|
||||
str(AudioAsset("winning_call").get_local_path()),
|
||||
]
|
||||
|
||||
# Run LID for each file ----------------------------------------------
|
||||
print(f"{'Audio File':<50} {'Language (sync)':<20} {'Language (stream)'}")
|
||||
print("-" * 90)
|
||||
|
||||
for path in audio_paths:
|
||||
basename = os.path.basename(path)
|
||||
|
||||
# 1) Synchronous via OpenAI SDK
|
||||
lang_sync = identify_language(path, client, model)
|
||||
|
||||
# 2) Streaming via raw HTTP
|
||||
lang_stream = identify_language_streaming(path, model, api_base)
|
||||
|
||||
print(f"{basename:<50} {lang_sync:<20} {lang_stream}")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="FireRedLID – Language Identification demo via vLLM",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio_paths",
|
||||
nargs="+",
|
||||
default=None,
|
||||
help=(
|
||||
"One or more audio files to identify. "
|
||||
"If omitted, uses vLLM's built-in sample audios."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api_base",
|
||||
type=str,
|
||||
default="http://localhost:8000/v1",
|
||||
help="vLLM API base URL (default: http://localhost:8000/v1)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -1,150 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This script demonstrates how to use the vLLM Realtime WebSocket API to perform
|
||||
audio transcription by uploading an audio file.
|
||||
|
||||
Before running this script, you must start the vLLM server with a realtime-capable
|
||||
model, for example:
|
||||
|
||||
vllm serve mistralai/Voxtral-Mini-4B-Realtime-2602 --enforce-eager
|
||||
|
||||
Requirements:
|
||||
- vllm with audio support
|
||||
- websockets
|
||||
- numpy
|
||||
|
||||
The script:
|
||||
1. Connects to the Realtime WebSocket endpoint
|
||||
2. Converts an audio file to PCM16 @ 16kHz
|
||||
3. Sends audio chunks to the server
|
||||
4. Receives and prints transcription as it streams
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import websockets
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
|
||||
def audio_to_pcm16_base64(audio_path: str) -> str:
|
||||
"""
|
||||
Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
|
||||
"""
|
||||
# Load audio and resample to 16kHz mono
|
||||
audio, _ = load_audio(audio_path, sr=16000, mono=True)
|
||||
# Convert to PCM16
|
||||
pcm16 = (audio * 32767).astype(np.int16)
|
||||
# Encode as base64
|
||||
return base64.b64encode(pcm16.tobytes()).decode("utf-8")
|
||||
|
||||
|
||||
async def realtime_transcribe(audio_path: str, host: str, port: int, model: str):
|
||||
"""
|
||||
Connect to the Realtime API and transcribe an audio file.
|
||||
"""
|
||||
uri = f"ws://{host}:{port}/v1/realtime"
|
||||
|
||||
async with websockets.connect(uri) as ws:
|
||||
# Wait for session.created
|
||||
response = json.loads(await ws.recv())
|
||||
if response["type"] == "session.created":
|
||||
print(f"Session created: {response['id']}")
|
||||
else:
|
||||
print(f"Unexpected response: {response}")
|
||||
return
|
||||
|
||||
# Validate model
|
||||
await ws.send(json.dumps({"type": "session.update", "model": model}))
|
||||
|
||||
# Signal ready to start
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
|
||||
|
||||
# Convert audio file to base64 PCM16
|
||||
print(f"Loading audio from: {audio_path}")
|
||||
audio_base64 = audio_to_pcm16_base64(audio_path)
|
||||
|
||||
# Send audio in chunks (4KB of raw audio = ~8KB base64)
|
||||
chunk_size = 4096
|
||||
audio_bytes = base64.b64decode(audio_base64)
|
||||
total_chunks = (len(audio_bytes) + chunk_size - 1) // chunk_size
|
||||
|
||||
print(f"Sending {total_chunks} audio chunks...")
|
||||
for i in range(0, len(audio_bytes), chunk_size):
|
||||
chunk = audio_bytes[i : i + chunk_size]
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(chunk).decode("utf-8"),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Signal all audio is sent
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
print("Audio sent. Waiting for transcription...\n")
|
||||
|
||||
# Receive transcription
|
||||
print("Transcription: ", end="", flush=True)
|
||||
while True:
|
||||
response = json.loads(await ws.recv())
|
||||
if response["type"] == "transcription.delta":
|
||||
print(response["delta"], end="", flush=True)
|
||||
elif response["type"] == "transcription.done":
|
||||
print(f"\n\nFinal transcription: {response['text']}")
|
||||
if response.get("usage"):
|
||||
print(f"Usage: {response['usage']}")
|
||||
break
|
||||
elif response["type"] == "error":
|
||||
print(f"\nError: {response['error']}")
|
||||
break
|
||||
|
||||
|
||||
def main(args):
|
||||
if args.audio_path:
|
||||
audio_path = args.audio_path
|
||||
else:
|
||||
# Use default audio asset
|
||||
audio_path = str(AudioAsset("mary_had_lamb").get_local_path())
|
||||
print(f"No audio path provided, using default: {audio_path}")
|
||||
|
||||
asyncio.run(realtime_transcribe(audio_path, args.host, args.port, args.model))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Realtime WebSocket Transcription Client"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="mistralai/Voxtral-Mini-4B-Realtime-2602",
|
||||
help="Model that is served and should be pinged.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio_path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to the audio file to transcribe.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="localhost",
|
||||
help="vLLM server host (default: localhost)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="vLLM server port (default: 8000)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -1,183 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API.
|
||||
|
||||
Start the vLLM server first:
|
||||
|
||||
vllm serve mistralai/Voxtral-Mini-4B-Realtime-2602 --enforce-eager
|
||||
|
||||
Then run this script:
|
||||
|
||||
python openai_realtime_microphone_client.py --host localhost --port 8000
|
||||
|
||||
Use --share to create a public Gradio link.
|
||||
|
||||
Requirements: websockets, numpy, gradio
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
|
||||
import gradio as gr
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import websockets
|
||||
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
# Global state
|
||||
audio_queue: queue.Queue = queue.Queue()
|
||||
transcription_text = ""
|
||||
is_running = False
|
||||
ws_url = ""
|
||||
model = ""
|
||||
|
||||
|
||||
async def websocket_handler():
|
||||
"""Connect to WebSocket and handle audio streaming + transcription."""
|
||||
global transcription_text, is_running
|
||||
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
# Wait for session.created
|
||||
await ws.recv()
|
||||
|
||||
# Validate model
|
||||
await ws.send(json.dumps({"type": "session.update", "model": model}))
|
||||
|
||||
# Signal ready
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
|
||||
|
||||
async def send_audio():
|
||||
while is_running:
|
||||
try:
|
||||
chunk = await asyncio.get_event_loop().run_in_executor(
|
||||
None, lambda: audio_queue.get(timeout=0.1)
|
||||
)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{"type": "input_audio_buffer.append", "audio": chunk}
|
||||
)
|
||||
)
|
||||
except queue.Empty:
|
||||
continue
|
||||
|
||||
async def receive_transcription():
|
||||
global transcription_text
|
||||
async for message in ws:
|
||||
data = json.loads(message)
|
||||
if data.get("type") == "transcription.delta":
|
||||
transcription_text += data["delta"]
|
||||
|
||||
await asyncio.gather(send_audio(), receive_transcription())
|
||||
|
||||
|
||||
def start_websocket():
|
||||
"""Start WebSocket connection in background thread."""
|
||||
global is_running
|
||||
is_running = True
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
loop.run_until_complete(websocket_handler())
|
||||
except Exception as e:
|
||||
print(f"WebSocket error: {e}")
|
||||
|
||||
|
||||
def start_recording():
|
||||
"""Start the transcription service."""
|
||||
global transcription_text
|
||||
transcription_text = ""
|
||||
thread = threading.Thread(target=start_websocket, daemon=True)
|
||||
thread.start()
|
||||
return gr.update(interactive=False), gr.update(interactive=True), ""
|
||||
|
||||
|
||||
def stop_recording():
|
||||
"""Stop the transcription service."""
|
||||
global is_running
|
||||
is_running = False
|
||||
return gr.update(interactive=True), gr.update(interactive=False), transcription_text
|
||||
|
||||
|
||||
def process_audio(audio):
|
||||
"""Process incoming audio and queue for streaming."""
|
||||
global transcription_text
|
||||
|
||||
if audio is None or not is_running:
|
||||
return transcription_text
|
||||
|
||||
sample_rate, audio_data = audio
|
||||
|
||||
# Convert to mono if stereo
|
||||
if len(audio_data.shape) > 1:
|
||||
audio_data = audio_data.mean(axis=1)
|
||||
|
||||
# Normalize to float
|
||||
if audio_data.dtype == np.int16:
|
||||
audio_float = audio_data.astype(np.float32) / 32767.0
|
||||
else:
|
||||
audio_float = audio_data.astype(np.float32)
|
||||
|
||||
# Resample to 16kHz if needed
|
||||
if sample_rate != SAMPLE_RATE:
|
||||
num_samples = int(len(audio_float) * SAMPLE_RATE / sample_rate)
|
||||
audio_float = np.interp(
|
||||
np.linspace(0, len(audio_float) - 1, num_samples),
|
||||
np.arange(len(audio_float)),
|
||||
audio_float,
|
||||
)
|
||||
|
||||
# Convert to PCM16 and base64 encode
|
||||
pcm16 = (audio_float * 32767).astype(np.int16)
|
||||
b64_chunk = base64.b64encode(pcm16.tobytes()).decode("utf-8")
|
||||
audio_queue.put(b64_chunk)
|
||||
|
||||
return transcription_text
|
||||
|
||||
|
||||
# Gradio interface
|
||||
with gr.Blocks(title="Real-time Speech Transcription") as demo:
|
||||
gr.Markdown("# Real-time Speech Transcription")
|
||||
gr.Markdown("Click **Start** and speak into your microphone.")
|
||||
|
||||
with gr.Row():
|
||||
start_btn = gr.Button("Start", variant="primary")
|
||||
stop_btn = gr.Button("Stop", variant="stop", interactive=False)
|
||||
|
||||
audio_input = gr.Audio(sources=["microphone"], streaming=True, type="numpy")
|
||||
transcription_output = gr.Textbox(label="Transcription", lines=5)
|
||||
|
||||
start_btn.click(
|
||||
start_recording, outputs=[start_btn, stop_btn, transcription_output]
|
||||
)
|
||||
stop_btn.click(stop_recording, outputs=[start_btn, stop_btn, transcription_output])
|
||||
audio_input.stream(
|
||||
process_audio, inputs=[audio_input], outputs=[transcription_output]
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Realtime WebSocket Transcription with Gradio"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="mistralai/Voxtral-Mini-4B-Realtime-2602",
|
||||
help="Model that is served and should be pinged.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host", type=str, default="localhost", help="vLLM server host"
|
||||
)
|
||||
parser.add_argument("--port", type=int, default=8000, help="vLLM server port")
|
||||
parser.add_argument(
|
||||
"--share", action="store_true", help="Create public Gradio link"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
ws_url = f"ws://{args.host}:{args.port}/v1/realtime"
|
||||
model = args.model
|
||||
demo.launch(share=args.share)
|
||||
@@ -1,44 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server.
|
||||
Reasoning models can be used through the Responses API as seen here
|
||||
https://platform.openai.com/docs/api-reference/responses
|
||||
For example:
|
||||
vllm serve Qwen/Qwen3-8B --reasoning-parser qwen3
|
||||
|
||||
"""
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
input_messages = [{"role": "user", "content": "What model are you?"}]
|
||||
|
||||
|
||||
def main():
|
||||
base_url = "http://localhost:8000/v1"
|
||||
client = OpenAI(base_url=base_url, api_key="empty")
|
||||
model = "Qwen/Qwen3-8B" # get_first_model(client)
|
||||
response = client.responses.create(
|
||||
model=model,
|
||||
input=input_messages,
|
||||
)
|
||||
|
||||
for message in response.output:
|
||||
if message.type == "reasoning":
|
||||
# append reasoning message
|
||||
input_messages.append(message)
|
||||
|
||||
response_2 = client.responses.create(
|
||||
model=model,
|
||||
input=input_messages,
|
||||
)
|
||||
print(response_2.output_text)
|
||||
# I am Qwen, a large language model developed by Alibaba Cloud.
|
||||
# I am designed to assist with a wide range of tasks, including
|
||||
# answering questions, creating content, coding, and engaging in
|
||||
# conversations. I can help with various topics and provide
|
||||
# information or support in multiple languages. How can I assist you today?
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,184 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example demonstrating MCP (Model Context Protocol) tools with the Responses API.
|
||||
|
||||
This example shows how to use MCP tools with different allowed_tools configurations:
|
||||
1. No filter (allows all tools from the MCP server)
|
||||
2. Wildcard "*" (explicitly allows all tools)
|
||||
3. Specific tool names (filters to only those tools)
|
||||
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with MCP tools enabled.
|
||||
For example:
|
||||
vllm serve openai/gpt-oss-20b --enforce-eager --tool-server demo
|
||||
|
||||
Environment variables:
|
||||
- VLLM_ENABLE_RESPONSES_API_STORE=1
|
||||
- VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=code_interpreter,container
|
||||
- VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS=1
|
||||
"""
|
||||
|
||||
from openai import OpenAI
|
||||
from utils import get_first_model
|
||||
|
||||
|
||||
def example_no_filter():
|
||||
"""Example with no allowed_tools filter - allows all tools."""
|
||||
print("=" * 60)
|
||||
print("Example 1: No allowed_tools filter (allows all tools)")
|
||||
print("=" * 60)
|
||||
|
||||
base_url = "http://0.0.0.0:8000/v1"
|
||||
client = OpenAI(base_url=base_url, api_key="empty")
|
||||
model = get_first_model(client)
|
||||
|
||||
response = client.responses.create(
|
||||
model=model,
|
||||
input="Execute this code: print('Hello from Python!')",
|
||||
instructions="Use the Python tool to execute code.",
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "code_interpreter",
|
||||
"server_url": "http://localhost:8888",
|
||||
# No allowed_tools specified - all tools are available
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(f"Status: {response.status}")
|
||||
print(f"Output: {response.output_text}")
|
||||
print()
|
||||
|
||||
|
||||
def example_wildcard():
|
||||
"""Example with allowed_tools=['*'] - explicitly allows all tools."""
|
||||
print("=" * 60)
|
||||
print("Example 2: allowed_tools=['*'] (select all tools)")
|
||||
print("=" * 60)
|
||||
|
||||
base_url = "http://0.0.0.0:8000/v1"
|
||||
client = OpenAI(base_url=base_url, api_key="empty")
|
||||
model = get_first_model(client)
|
||||
|
||||
response = client.responses.create(
|
||||
model=model,
|
||||
input="Execute this code: print('Hello from Python with wildcard!')",
|
||||
instructions="Use the Python tool to execute code.",
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "code_interpreter",
|
||||
"server_url": "http://localhost:8888",
|
||||
# Using "*" to explicitly allow all tools from this MCP server
|
||||
# This is equivalent to not specifying allowed_tools
|
||||
"allowed_tools": ["*"],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(f"Status: {response.status}")
|
||||
print(f"Output: {response.output_text}")
|
||||
print()
|
||||
|
||||
|
||||
def example_specific_tools():
|
||||
"""Example with specific allowed_tools list - filters available tools.
|
||||
|
||||
Note: This example uses 'web_search_preview' (browser) which has multiple
|
||||
sub-tools: 'search', 'open', 'find'. The code_interpreter (python) doesn't
|
||||
have sub-tools, so filtering doesn't apply there.
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("Example 3: allowed_tools=['search'] (filter browser to specific tools)")
|
||||
print("=" * 60)
|
||||
|
||||
base_url = "http://0.0.0.0:8000/v1"
|
||||
client = OpenAI(base_url=base_url, api_key="empty")
|
||||
model = get_first_model(client)
|
||||
|
||||
response = client.responses.create(
|
||||
model=model,
|
||||
input="Search for 'Python programming tutorials'",
|
||||
instructions="Use the browser tool to search.",
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "web_search_preview",
|
||||
"server_url": "http://localhost:8888",
|
||||
# Browser has tools: 'search', 'open', 'find'
|
||||
# Only allow 'search' - blocks 'open' and 'find'
|
||||
"allowed_tools": ["search"],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(f"Status: {response.status}")
|
||||
print(f"Output: {response.output_text}")
|
||||
print()
|
||||
|
||||
|
||||
def example_object_format():
|
||||
"""Example using object format for allowed_tools with browser tools."""
|
||||
print("=" * 60)
|
||||
print("Example 4: allowed_tools with object format")
|
||||
print("=" * 60)
|
||||
|
||||
base_url = "http://0.0.0.0:8000/v1"
|
||||
client = OpenAI(base_url=base_url, api_key="empty")
|
||||
model = get_first_model(client)
|
||||
|
||||
response = client.responses.create(
|
||||
model=model,
|
||||
input="Search for 'machine learning' and open the first result",
|
||||
instructions="Use the browser tool.",
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "web_search_preview",
|
||||
"server_url": "http://localhost:8888",
|
||||
# Object format with tool_names field
|
||||
# Can also include read_only and other fields
|
||||
# Browser has tools: 'search', 'open', 'find'
|
||||
"allowed_tools": {
|
||||
"tool_names": [
|
||||
"search",
|
||||
"open",
|
||||
], # Allow search and open, block find
|
||||
"read_only": False,
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(f"Status: {response.status}")
|
||||
print(f"Output: {response.output_text}")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all examples."""
|
||||
print("\n" + "=" * 60)
|
||||
print("MCP Tools with allowed_tools Examples")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
# Run all examples
|
||||
example_no_filter()
|
||||
example_wildcard()
|
||||
example_specific_tools()
|
||||
example_object_format()
|
||||
|
||||
print("=" * 60)
|
||||
print("Summary:")
|
||||
print(" - No filter or '*' → All tools available from server")
|
||||
print(" - Specific list → Only those sub-tools available")
|
||||
print(" - Object format → More control with tool_names field")
|
||||
print("")
|
||||
print("Note: allowed_tools filters SUB-TOOLS within an MCP server:")
|
||||
print(" - code_interpreter (python): No sub-tools to filter")
|
||||
print(" - web_search_preview (browser): Has 'search', 'open', 'find'")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled.
|
||||
Reasoning models can be used through the Responses API as seen here
|
||||
https://platform.openai.com/docs/api-reference/responses
|
||||
For example:
|
||||
vllm serve Qwen/Qwen3-1.7B --reasoning-parser qwen3 \
|
||||
--structured-outputs-config.backend xgrammar \
|
||||
--enable-auto-tool-choice --tool-call-parser hermes
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from openai import OpenAI
|
||||
from utils import get_first_model
|
||||
|
||||
|
||||
def get_weather(latitude: float, longitude: float) -> str:
|
||||
"""
|
||||
Mock function to simulate getting weather data.
|
||||
In a real application, this would call an external weather API.
|
||||
"""
|
||||
return f"Current temperature at ({latitude}, {longitude}) is 20°C."
|
||||
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Get current temperature for provided coordinates in celsius.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"latitude": {"type": "number"},
|
||||
"longitude": {"type": "number"},
|
||||
},
|
||||
"required": ["latitude", "longitude"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": True,
|
||||
}
|
||||
]
|
||||
|
||||
input_messages = [
|
||||
{"role": "user", "content": "What's the weather like in Paris today?"}
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
base_url = "http://0.0.0.0:8000/v1"
|
||||
client = OpenAI(base_url=base_url, api_key="empty")
|
||||
model = get_first_model(client)
|
||||
response = client.responses.create(
|
||||
model=model, input=input_messages, tools=tools, tool_choice="required"
|
||||
)
|
||||
|
||||
for out in response.output:
|
||||
if out.type == "function_call":
|
||||
print("Function call:", out.name, out.arguments)
|
||||
tool_call = out
|
||||
args = json.loads(tool_call.arguments)
|
||||
result = get_weather(args["latitude"], args["longitude"])
|
||||
|
||||
input_messages.append(tool_call) # append model's function call message
|
||||
input_messages.append(
|
||||
{ # append result message
|
||||
"type": "function_call_output",
|
||||
"call_id": tool_call.call_id,
|
||||
"output": str(result),
|
||||
}
|
||||
)
|
||||
response_2 = client.responses.create(
|
||||
model=model,
|
||||
input=input_messages,
|
||||
tools=tools,
|
||||
)
|
||||
print(response_2.output_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,197 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This script demonstrates how to use the vLLM API server to perform audio
|
||||
transcription with the `openai/whisper-large-v3` model.
|
||||
|
||||
Before running this script, you must start the vLLM server with the following command:
|
||||
|
||||
vllm serve openai/whisper-large-v3
|
||||
|
||||
Requirements:
|
||||
- vLLM with audio support
|
||||
- openai Python SDK
|
||||
- httpx for streaming support
|
||||
|
||||
The script performs:
|
||||
1. Synchronous transcription using OpenAI-compatible API.
|
||||
2. Streaming transcription using raw HTTP request to the vLLM server.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
|
||||
def sync_openai(
|
||||
audio_path: str,
|
||||
client: OpenAI,
|
||||
model: str,
|
||||
*,
|
||||
repetition_penalty: float = 1.3,
|
||||
hotwords: str = None,
|
||||
):
|
||||
"""
|
||||
Perform synchronous transcription using OpenAI-compatible API.
|
||||
"""
|
||||
with open(audio_path, "rb") as f:
|
||||
transcription = client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=model,
|
||||
language="en",
|
||||
response_format="json",
|
||||
temperature=0.0,
|
||||
# Additional sampling params not provided by OpenAI API.
|
||||
extra_body=dict(
|
||||
seed=4419,
|
||||
repetition_penalty=repetition_penalty,
|
||||
hotwords=hotwords,
|
||||
),
|
||||
)
|
||||
print("transcription result [sync]:", transcription.text)
|
||||
|
||||
|
||||
async def stream_openai_response(
|
||||
audio_path: str, client: AsyncOpenAI, model: str, hotwords: str = None
|
||||
):
|
||||
"""
|
||||
Perform asynchronous transcription using OpenAI-compatible API.
|
||||
"""
|
||||
print("\ntranscription result [stream]:", end=" ")
|
||||
with open(audio_path, "rb") as f:
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=model,
|
||||
language="en",
|
||||
response_format="json",
|
||||
temperature=0.0,
|
||||
# Additional sampling params not provided by OpenAI API.
|
||||
extra_body=dict(
|
||||
seed=420,
|
||||
top_p=0.6,
|
||||
hotwords=hotwords,
|
||||
),
|
||||
stream=True,
|
||||
)
|
||||
async for chunk in transcription:
|
||||
if chunk.choices:
|
||||
content = chunk.choices[0].get("delta", {}).get("content")
|
||||
print(content, end="", flush=True)
|
||||
|
||||
print() # Final newline after stream ends
|
||||
|
||||
|
||||
def stream_api_response(audio_path: str, model: str, openai_api_base: str):
|
||||
"""
|
||||
Perform streaming transcription using raw HTTP requests to the vLLM API server.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
api_url = f"{openai_api_base}/audio/transcriptions"
|
||||
headers = {"User-Agent": "Transcription-Client"}
|
||||
with open(audio_path, "rb") as f:
|
||||
files = {"file": (os.path.basename(audio_path), f)}
|
||||
data = {
|
||||
"stream": "true",
|
||||
"model": model,
|
||||
"language": "en",
|
||||
"response_format": "json",
|
||||
}
|
||||
|
||||
print("\ntranscription result [stream]:", end=" ")
|
||||
response = requests.post(
|
||||
api_url, headers=headers, files=files, data=data, stream=True
|
||||
)
|
||||
for chunk in response.iter_lines(
|
||||
chunk_size=8192, decode_unicode=False, delimiter=b"\n"
|
||||
):
|
||||
if chunk:
|
||||
data = chunk[len("data: ") :]
|
||||
data = json.loads(data.decode("utf-8"))
|
||||
data = data["choices"][0]
|
||||
delta = data["delta"]["content"]
|
||||
print(delta, end="", flush=True)
|
||||
|
||||
finish_reason = data.get("finish_reason")
|
||||
if finish_reason is not None:
|
||||
print(f"\n[Stream finished reason: {finish_reason}]")
|
||||
break
|
||||
|
||||
|
||||
def main(args):
|
||||
mary_had_lamb = str(AudioAsset("mary_had_lamb").get_local_path())
|
||||
winning_call = str(AudioAsset("winning_call").get_local_path())
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
client = OpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
model = client.models.list().data[0].id
|
||||
print(f"Using model: {model}")
|
||||
|
||||
# Run the synchronous function
|
||||
sync_openai(
|
||||
audio_path=args.audio_path if args.audio_path else mary_had_lamb,
|
||||
client=client,
|
||||
model=model,
|
||||
repetition_penalty=args.repetition_penalty,
|
||||
hotwords=args.hotwords,
|
||||
)
|
||||
|
||||
# Run the asynchronous function
|
||||
if "openai" in model:
|
||||
client = AsyncOpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
asyncio.run(
|
||||
stream_openai_response(
|
||||
args.audio_path if args.audio_path else winning_call,
|
||||
client,
|
||||
model,
|
||||
hotwords=args.hotwords,
|
||||
)
|
||||
)
|
||||
else:
|
||||
stream_api_response(
|
||||
args.audio_path if args.audio_path else winning_call,
|
||||
model,
|
||||
openai_api_base,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# setup argparser
|
||||
parser = argparse.ArgumentParser(
|
||||
description="OpenAI Transcription Client using vLLM API Server"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio_path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="The path to the audio file to transcribe.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repetition_penalty",
|
||||
type=float,
|
||||
default=1.3,
|
||||
help="repetition penalty",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hotwords",
|
||||
type=str,
|
||||
default=None,
|
||||
help="hotwords",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from openai import OpenAI
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
|
||||
def sync_openai(audio_path: str, client: OpenAI, model: str):
|
||||
with open(audio_path, "rb") as f:
|
||||
translation = client.audio.translations.create(
|
||||
file=f,
|
||||
model=model,
|
||||
response_format="json",
|
||||
temperature=0.0,
|
||||
# Additional params not provided by OpenAI API.
|
||||
extra_body=dict(
|
||||
language="it",
|
||||
seed=4419,
|
||||
repetition_penalty=1.3,
|
||||
),
|
||||
)
|
||||
print("translation result:", translation.text)
|
||||
|
||||
|
||||
async def stream_openai_response(
|
||||
audio_path: str, base_url: str, api_key: str, model: str
|
||||
):
|
||||
data = {
|
||||
"language": "it",
|
||||
"stream": True,
|
||||
"model": model,
|
||||
}
|
||||
url = base_url + "/audio/translations"
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
print("translation result:", end=" ")
|
||||
# OpenAI translation API client does not support streaming.
|
||||
async with httpx.AsyncClient() as client:
|
||||
with open(audio_path, "rb") as f:
|
||||
async with client.stream(
|
||||
"POST", url, files={"file": f}, data=data, headers=headers
|
||||
) as response:
|
||||
async for line in response.aiter_lines():
|
||||
# Each line is a JSON object prefixed with 'data: '
|
||||
if line:
|
||||
if line.startswith("data: "):
|
||||
line = line[len("data: ") :]
|
||||
# Last chunk, stream ends
|
||||
if line.strip() == "[DONE]":
|
||||
break
|
||||
# Parse the JSON response
|
||||
chunk = json.loads(line)
|
||||
# Extract and print the content
|
||||
content = chunk["choices"][0].get("delta", {}).get("content")
|
||||
print(content, end="")
|
||||
|
||||
|
||||
def main():
|
||||
foscolo = str(AudioAsset("azacinto_foscolo").get_local_path())
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8000/v1"
|
||||
client = OpenAI(
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
model = client.models.list().data[0].id
|
||||
print(f"Using model: {model}")
|
||||
|
||||
sync_openai(foscolo, client, model)
|
||||
# Run the asynchronous function
|
||||
asyncio.run(stream_openai_response(foscolo, openai_api_base, openai_api_key, model))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import httpx
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
GEN_ENDPOINT = "http://localhost:8000/inference/v1/generate"
|
||||
DUMMY_API_KEY = "empty"
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
transport = httpx.HTTPTransport()
|
||||
headers = {"Authorization": f"Bearer {DUMMY_API_KEY}"}
|
||||
client = httpx.Client(
|
||||
transport=transport,
|
||||
base_url=GEN_ENDPOINT,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
)
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "How many countries are in the EU?"},
|
||||
]
|
||||
|
||||
|
||||
def main(client):
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
return_dict=True,
|
||||
).input_ids
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"max_tokens": 24, "temperature": 0.2, "detokenize": False},
|
||||
"stream": False,
|
||||
}
|
||||
resp = client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
print(data)
|
||||
print("-" * 50)
|
||||
print("Token generation results:")
|
||||
res = tokenizer.decode(data["choices"][0]["token_ids"])
|
||||
print(res)
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(client)
|
||||
Reference in New Issue
Block a user