[Doc] ruff format some Python examples (#26767)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-10-14 03:21:53 -07:00
committed by GitHub
parent 70b1b330e1
commit ef9676a1f1
20 changed files with 341 additions and 290 deletions
+4 -7
View File
@@ -32,7 +32,7 @@ the third parameter is the path to the LoRA adapter.
sampling_params = SamplingParams(
temperature=0,
max_tokens=256,
stop=["[/assistant]"]
stop=["[/assistant]"],
)
prompts = [
@@ -43,7 +43,7 @@ the third parameter is the path to the LoRA adapter.
outputs = llm.generate(
prompts,
sampling_params,
lora_request=LoRARequest("sql_adapter", 1, sql_lora_path)
lora_request=LoRARequest("sql_adapter", 1, sql_lora_path),
)
```
@@ -197,7 +197,7 @@ Alternatively, follow these example steps to implement your own plugin:
lora_request = LoRARequest(
lora_name=lora_name,
lora_path=local_path,
lora_int_id=abs(hash(lora_name))
lora_int_id=abs(hash(lora_name)),
)
return lora_request
```
@@ -296,10 +296,7 @@ To this end, we allow registration of default multimodal LoRAs to handle this au
if has_audio:
question = f"<|audio|>{question}"
chat = [
{
"role": "user",
"content": question
}
{"role": "user", "content": question},
]
return tokenizer.apply_chat_template(chat, tokenize=False)
+147 -130
View File
@@ -154,9 +154,7 @@ To substitute multiple images inside the same text prompt, you can pass in a lis
outputs = llm.generate({
"prompt": prompt,
"multi_modal_data": {
"image": [image1, image2]
},
"multi_modal_data": {"image": [image1, image2]},
})
for o in outputs:
@@ -183,21 +181,24 @@ conversation = [
{"role": "assistant", "content": "Hello! How can I assist you today?"},
{
"role": "user",
"content": [{
"type": "image_url",
"image_url": {
"url": image_url
}
},{
"type": "image_pil",
"image_pil": image_pil
}, {
"type": "image_embeds",
"image_embeds": image_embeds
}, {
"type": "text",
"text": "What's in these images?"
}],
"content": [
{
"type": "image_url",
"image_url": {"url": image_url},
},
{
"type": "image_pil",
"image_pil": image_pil,
},
{
"type": "image_embeds",
"image_embeds": image_embeds,
},
{
"type": "text",
"text": "What's in these images?",
},
],
},
]
@@ -224,7 +225,10 @@ Multi-image input can be extended to perform video captioning. We show this with
message = {
"role": "user",
"content": [
{"type": "text", "text": "Describe this set of frames. Consider the frames to be a part of the same video."},
{
"type": "text",
"text": "Describe this set of frames. Consider the frames to be a part of the same video.",
},
],
}
for i in range(len(video_frames)):
@@ -255,13 +259,13 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f
# Custom black background for dark theme
llm = LLM(
model="llava-hf/llava-1.5-7b-hf",
media_io_kwargs={"image": {"rgba_background_color": [0, 0, 0]}}
media_io_kwargs={"image": {"rgba_background_color": [0, 0, 0]}},
)
# Custom brand color background (e.g., blue)
llm = LLM(
model="llava-hf/llava-1.5-7b-hf",
media_io_kwargs={"image": {"rgba_background_color": [0, 0, 255]}}
media_io_kwargs={"image": {"rgba_background_color": [0, 0, 255]}},
)
```
@@ -294,20 +298,23 @@ Instead of NumPy arrays, you can also pass `'torch.Tensor'` instances, as shown
limit_mm_per_prompt={"video": 1},
)
sampling_params = SamplingParams(
max_tokens=1024,
)
sampling_params = SamplingParams(max_tokens=1024)
video_messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": [
{
"role": "system",
"content": "You are a helpful assistant.",
},
{
"role": "user",
"content": [
{"type": "text", "text": "describe this video."},
{
"type": "video",
"video": video_path,
"total_pixels": 20480 * 28 * 28,
"min_pixels": 16 * 28 * 28
}
"min_pixels": 16 * 28 * 28,
},
]
},
]
@@ -465,21 +472,24 @@ Then, you can use the OpenAI client as follows:
chat_response = client.chat.completions.create(
model="microsoft/Phi-3.5-vision-instruct",
messages=[{
"role": "user",
"content": [
# NOTE: The prompt formatting with the image token `<image>` is not needed
# since the prompt will be processed automatically by the API server.
{"type": "text", "text": "What’s in this image?"},
{
"type": "image_url",
"image_url": {
url": image_url
messages=[
{
"role": "user",
"content": [
# NOTE: The prompt formatting with the image token `<image>` is not needed
# since the prompt will be processed automatically by the API server.
{
"type": "text",
"text": "What’s in this image?",
},
"uuid": image_url # Optional
},
],
}],
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": image_url, # Optional
},
],
}
],
)
print("Chat completion output:", chat_response.choices[0].message.content)
@@ -489,26 +499,27 @@ Then, you can use the OpenAI client as follows:
chat_response = client.chat.completions.create(
model="microsoft/Phi-3.5-vision-instruct",
messages=[{
"role": "user",
"content": [
{"type": "text", "text": "What are the animals in these images?"},
{
"type": "image_url",
"image_url": {
"url": image_url_duck
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What are the animals in these images?",
},
"uuid": image_url_duck # Optional
},
{
"type": "image_url",
"image_url": {
"url": image_url_lion
{
"type": "image_url",
"image_url": {"url": image_url_duck},
"uuid": image_url_duck, # Optional
},
"uuid": image_url_lion # Optional
},
],
}],
{
"type": "image_url",
"image_url": {"url": image_url_lion},
"uuid": image_url_lion, # Optional
},
],
}
],
)
print("Chat completion output:", chat_response.choices[0].message.content)
```
@@ -560,23 +571,22 @@ Then, you can use the OpenAI client as follows:
## Use video url in the payload
chat_completion_from_url = client.chat.completions.create(
messages=[{
"role":
"user",
"content": [
{
"type": "text",
"text": "What's in this video?"
},
{
"type": "video_url",
"video_url": {
"url": video_url
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this video?",
},
"uuid": video_url # Optional
},
],
}],
{
"type": "video_url",
"video_url": {"url": video_url},
"uuid": video_url, # Optional
},
],
}
],
model=model,
max_completion_tokens=64,
)
@@ -652,23 +662,25 @@ Then, you can use the OpenAI client as follows:
audio_base64 = encode_base64_content_from_url(audio_url)
chat_completion_from_base64 = client.chat.completions.create(
messages=[{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this audio?"
},
{
"type": "input_audio",
"input_audio": {
"data": audio_base64,
"format": "wav"
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this audio?",
},
"uuid": audio_url # Optional
},
],
}],
{
"type": "input_audio",
"input_audio": {
"data": audio_base64,
"format": "wav",
},
"uuid": audio_url, # Optional
},
],
},
],
model=model,
max_completion_tokens=64,
)
@@ -683,22 +695,22 @@ Alternatively, you can pass `audio_url`, which is the audio counterpart of `imag
```python
chat_completion_from_url = client.chat.completions.create(
messages=[{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this audio?"
},
{
"type": "audio_url",
"audio_url": {
"url": audio_url
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this audio?",
},
"uuid": audio_url # Optional
},
],
}],
{
"type": "audio_url",
"audio_url": {"url": audio_url},
"uuid": audio_url, # Optional
},
],
}
],
model=model,
max_completion_tokens=64,
)
@@ -747,43 +759,48 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
# Basic usage - this is equivalent to the LLaVA example for offline inference
model = "llava-hf/llava-1.5-7b-hf"
embeds = {
embeds = {
"type": "image_embeds",
"image_embeds": f"{base64_image_embedding}",
"uuid": image_url # Optional
"uuid": image_url, # Optional
}
# Pass additional parameters (available to Qwen2-VL and MiniCPM-V)
model = "Qwen/Qwen2-VL-2B-Instruct"
embeds = {
embeds = {
"type": "image_embeds",
"image_embeds": {
"image_embeds": f"{base64_image_embedding}" , # Required
"image_grid_thw": f"{base64_image_grid_thw}" # Required by Qwen/Qwen2-VL-2B-Instruct
"image_embeds": f"{base64_image_embedding}", # Required
"image_grid_thw": f"{base64_image_grid_thw}", # Required by Qwen/Qwen2-VL-2B-Instruct
},
"uuid": image_url # Optional
"uuid": image_url, # Optional
}
model = "openbmb/MiniCPM-V-2_6"
embeds = {
embeds = {
"type": "image_embeds",
"image_embeds": {
"image_embeds": f"{base64_image_embedding}" , # Required
"image_sizes": f"{base64_image_sizes}" # Required by openbmb/MiniCPM-V-2_6
"image_embeds": f"{base64_image_embedding}", # Required
"image_sizes": f"{base64_image_sizes}", # Required by openbmb/MiniCPM-V-2_6
},
"uuid": image_url # Optional
"uuid": image_url, # Optional
}
chat_completion = client.chat.completions.create(
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": [
{
"type": "text",
"text": "What's in this image?",
"role": "system",
"content": "You are a helpful assistant.",
},
embeds,
],
},
],
{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this image?",
},
embeds,
],
},
],
model=model,
)
```
@@ -802,22 +819,22 @@ For Online Serving, you can also skip sending media if you expect cache hits wit
{
"type": "image_embeds",
"image_embeds": None,
"uuid": image_uuid
"uuid": image_uuid,
},
# input_audio:
{
"type": "input_audio",
"input_audio": None,
"uuid": audio_uuid
"uuid": audio_uuid,
},
# PIL Image:
{
"type": "image_pil",
"image_pil": None
"uuid": image_uuid
}
"image_pil": None,
"uuid": image_uuid,
},
```
+31 -25
View File
@@ -117,9 +117,11 @@ OpenAI Python client library does not officially support `reasoning_content` att
# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
# For Qwen3 series, if you want to disable thinking in reasoning mode, add:
# extra_body={"chat_template_kwargs": {"enable_thinking": False}}
stream = client.chat.completions.create(model=model,
messages=messages,
stream=True)
stream = client.chat.completions.create(
model=model,
messages=messages,
stream=True,
)
print("client: Start streaming chat completions...")
printed_reasoning_content = False
@@ -159,27 +161,29 @@ The reasoning content is also available when both tool calling and the reasoning
client = OpenAI(base_url="http://localhost:8000/v1", api_key="dummy")
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["location", "unit"]
}
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
"required": ["location", "unit"],
}
},
}
}]
]
response = client.chat.completions.create(
model=client.models.list().data[0].id,
messages=[{"role": "user", "content": "What's the weather like in San Francisco?"}],
tools=tools,
tool_choice="auto"
tool_choice="auto",
)
print(response)
@@ -225,7 +229,7 @@ You can add a new `ReasoningParser` similar to <gh-file:vllm/reasoning/deepseek_
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Instance method that should be implemented for extracting reasoning
from an incomplete response; for use when handling reasoning calls and
@@ -235,8 +239,10 @@ You can add a new `ReasoningParser` similar to <gh-file:vllm/reasoning/deepseek_
"""
def extract_reasoning_content(
self, model_output: str, request: ChatCompletionRequest
) -> tuple[Optional[str], Optional[str]]:
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
) -> tuple[str | None, str | None]:
"""
Extract reasoning content from a complete model-generated string.
@@ -274,10 +280,10 @@ Additionally, to enable structured output, you'll need to create a new `Reasoner
@classmethod
def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
return cls(start_token_id=tokenizer.encode(
"<think>", add_special_tokens=False)[0],
end_token_id=tokenizer.encode("</think>",
add_special_tokens=False)[0])
return cls(
start_token_id=tokenizer.encode("<think>", add_special_tokens=False)[0],
end_token_id=tokenizer.encode("</think>", add_special_tokens=False)[0],
)
def is_reasoning_end(self, input_ids: list[int]) -> bool:
return self.end_token_id in input_ids
+19 -18
View File
@@ -27,27 +27,29 @@ Next, make a request that triggers the model to use the available tools:
return f"Getting the weather for {location} in {unit}..."
tool_functions = {"get_weather": get_weather}
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string", "description": "City and state, e.g., 'San Francisco, CA'"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["location", "unit"],
},
"required": ["location", "unit"]
}
}
}]
},
},
]
response = client.chat.completions.create(
model=client.models.list().data[0].id,
messages=[{"role": "user", "content": "What's the weather like in San Francisco?"}],
tools=tools,
tool_choice="auto"
tool_choice="auto",
)
tool_call = response.choices[0].message.tool_calls[0].function
@@ -402,8 +404,7 @@ Here is a summary of a plugin file:
# adjust request. e.g.: set skip special tokens
# to False for tool call output.
def adjust_request(
self, request: ChatCompletionRequest) -> ChatCompletionRequest:
def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
return request
# implement the tool call parse for stream call
@@ -416,7 +417,7 @@ Here is a summary of a plugin file:
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
return delta
# implement the tool parse for non-stream call