Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
NOTE: This API server is used only for demonstrating usage of AsyncEngine
"""NOTE: This API server is used only for demonstrating usage of AsyncEngine
and simple performance benchmarks. It is not intended for production use.
For production use, we recommend using our OpenAI compatible server.
We are also not going to accept PRs modifying this file.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
vLLM Chat Assistant - A Streamlit Web Interface
"""vLLM Chat Assistant - A Streamlit Web Interface.
A streamlined chat interface that quickly integrates
with vLLM API server.
@@ -78,6 +77,7 @@ def create_new_chat_session():
- current_session: Sets to new session ID
- active_session: Sets to new session ID
- messages: Resets to empty list
"""
session_id = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
st.session_state.sessions[session_id] = []
@@ -101,6 +101,7 @@ def switch_to_chat_session(session_id):
- current_session: Updated to specified session_id
- active_session: Updated to specified session_id
- messages: Loaded from sessions[session_id]
"""
st.session_state.current_session = session_id
st.session_state.active_session = session_id
@@ -134,6 +135,7 @@ def get_llm_response(messages, model, reason, content_ph=None, reasoning_ph=None
Note:
The function uses streamlit placeholders for live updates.
When reason=True, the reasoning process appears above the response.
"""
full_text = ""
think_text = ""
@@ -256,6 +258,7 @@ def server_supports_reasoning():
Returns:
bool: True if the model supports reasoning, False otherwise
"""
resp = client.chat.completions.create(
model=model,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Retrieval Augmented Generation (RAG) Implementation with Langchain
"""Retrieval Augmented Generation (RAG) Implementation with Langchain
==================================================================
This script demonstrates a RAG implementation using LangChain, Milvus
@@ -35,6 +34,7 @@ Notes:
- Ensure both vLLM services are running before executing
- Default ports: 8000 (embedding), 8001 (chat)
- First run may take time to download models
"""
import argparse
@@ -52,9 +52,7 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter
def load_and_split_documents(config: dict[str, Any]):
"""
Load and split documents from web URL
"""
"""Load and split documents from web URL."""
try:
loader = WebBaseLoader(web_paths=(config["url"],))
docs = loader.load()
@@ -70,9 +68,7 @@ def load_and_split_documents(config: dict[str, Any]):
def init_vectorstore(config: dict[str, Any], documents: list[Document]):
"""
Initialize vector store with documents
"""
"""Initialize vector store with documents."""
return Milvus.from_documents(
documents=documents,
embedding=OpenAIEmbeddings(
@@ -86,9 +82,7 @@ def init_vectorstore(config: dict[str, Any], documents: list[Document]):
def init_llm(config: dict[str, Any]):
"""
Initialize llm
"""
"""Initialize llm."""
return ChatOpenAI(
model=config["chat_model"],
openai_api_key=config["vllm_api_key"],
@@ -97,9 +91,7 @@ def init_llm(config: dict[str, Any]):
def get_qa_prompt():
"""
Get question answering prompt template
"""
"""Get question answering prompt template."""
template = """You are an assistant for question-answering tasks.
Use the following pieces of retrieved context to answer the question.
If you don't know the answer, just say that you don't know.
@@ -112,16 +104,12 @@ Answer:
def format_docs(docs: list[Document]):
"""
Format documents for prompt
"""
"""Format documents for prompt."""
return "\n\n".join(doc.page_content for doc in docs)
def create_qa_chain(retriever: Any, llm: ChatOpenAI, prompt: PromptTemplate):
"""
Set up question answering chain
"""
"""Set up question answering chain."""
return (
{
"context": retriever | format_docs,
@@ -134,9 +122,7 @@ def create_qa_chain(retriever: Any, llm: ChatOpenAI, prompt: PromptTemplate):
def get_parser() -> argparse.ArgumentParser:
"""
Parse command line arguments
"""
"""Parse command line arguments."""
parser = argparse.ArgumentParser(description="RAG with vLLM and langchain")
# Add command line arguments
@@ -192,10 +178,7 @@ def get_parser() -> argparse.ArgumentParser:
def init_config(args: Namespace):
"""
Initialize configuration settings from command line arguments
"""
"""Initialize configuration settings from command line arguments."""
return {
"vllm_api_key": args.vllm_api_key,
"vllm_embedding_endpoint": args.vllm_embedding_endpoint,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
RAG (Retrieval Augmented Generation) Implementation with LlamaIndex
"""RAG (Retrieval Augmented Generation) Implementation with LlamaIndex
================================================================
This script demonstrates a RAG system using:
@@ -35,6 +34,7 @@ Notes:
- Ensure both vLLM services are running before executing
- Default ports: 8000 (embedding), 8001 (chat)
- First run may take time to download models
"""
import argparse
@@ -50,7 +50,7 @@ from llama_index.vector_stores.milvus import MilvusVectorStore
def init_config(args: Namespace):
"""Initialize configuration with command line arguments"""
"""Initialize configuration with command line arguments."""
return {
"url": args.url,
"embedding_model": args.embedding_model,
@@ -66,12 +66,12 @@ def init_config(args: Namespace):
def load_documents(url: str) -> list:
"""Load and process web documents"""
"""Load and process web documents."""
return SimpleWebPageReader(html_to_text=True).load_data([url])
def setup_models(config: dict[str, Any]):
"""Configure embedding and chat models"""
"""Configure embedding and chat models."""
Settings.embed_model = OpenAILikeEmbedding(
api_base=config["embedding_endpoint"],
api_key=config["vllm_api_key"],
@@ -96,14 +96,14 @@ def setup_models(config: dict[str, Any]):
def setup_vector_store(db_path: str) -> MilvusVectorStore:
"""Initialize vector store"""
"""Initialize vector store."""
sample_emb = Settings.embed_model.get_text_embedding("test")
print(f"Embedding dimension: {len(sample_emb)}")
return MilvusVectorStore(uri=db_path, dim=len(sample_emb), overwrite=True)
def create_index(documents: list, vector_store: MilvusVectorStore):
"""Create document index"""
"""Create document index."""
storage_context = StorageContext.from_defaults(vector_store=vector_store)
return VectorStoreIndex.from_documents(
documents,
@@ -112,13 +112,13 @@ def create_index(documents: list, vector_store: MilvusVectorStore):
def query_document(index: VectorStoreIndex, question: str, top_k: int):
"""Query document with given question"""
"""Query document with given question."""
query_engine = index.as_query_engine(similarity_top_k=top_k)
return query_engine.query(question)
def get_parser() -> argparse.ArgumentParser:
"""Parse command line arguments"""
"""Parse command line arguments."""
parser = argparse.ArgumentParser(description="RAG with vLLM and LlamaIndex")
# Add command line arguments
+2 -4
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Simple example demonstrating streaming offline inference with AsyncLLM (V1 engine).
"""Simple example demonstrating streaming offline inference with AsyncLLM (V1 engine).
This script shows the core functionality of vLLM's AsyncLLM engine for streaming
token-by-token output in offline inference scenarios. It demonstrates DELTA mode
@@ -20,8 +19,7 @@ from vllm.v1.engine.async_llm import AsyncLLM
async def stream_response(engine: AsyncLLM, prompt: str, request_id: str) -> None:
"""
Stream response from AsyncLLM and display tokens as they arrive.
"""Stream response from AsyncLLM and display tokens as they arrive.
This function demonstrates the core streaming pattern:
1. Create SamplingParams with DELTA output kind
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file demonstrates using the `LLMEngine`
"""This file demonstrates using the `LLMEngine`
for processing prompts with various sampling parameters.
"""
@@ -1,8 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
disagg_encoder_proxy.py
"""disagg_encoder_proxy.py
Proxy that routes OpenAI-compatible “/v1/chat/completions” requests to two
clusters:
@@ -200,8 +199,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
def extract_mm_items(request_data: dict) -> list[dict]:
"""
Return *all* image/audio/video items that appear anywhere in `messages`.
"""Return *all* image/audio/video items that appear anywhere in `messages`.
Each returned dict looks like:
{ "type": "image_url", "image_url": {...} }
@@ -224,8 +222,7 @@ async def fanout_encoder_primer(
req_id: str,
consumer_zmq: str | None = None,
) -> tuple[dict[int, dict], dict[str, Any]]:
"""
1. Build one request *per MM item* with all text removed.
"""1. Build one request *per MM item* with all text removed.
2. Send them concurrently to the encode cluster.
3. Raise if any of them fails.
@@ -402,8 +399,7 @@ async def maybe_prefill(
p_url: str,
req_id: str,
) -> dict:
"""
- Do prefill-only task if p_url exist;
"""- Do prefill-only task if p_url exist;
- Return a new body carrying kv transfer params (for nixl connector)
- Else, skip and return the original request data for decode
@@ -849,14 +845,14 @@ async def _post_if_available(
payload: dict,
headers: dict,
) -> dict | None:
"""
POST `payload` to `url`.
"""POST `payload` to `url`.
Returns
-------
• The decoded JSON body on success (2xx)
• None if the endpoint does not exist (404)
• Raises for anything else.
"""
try:
resp = await session.post(url, json=payload, headers=headers)
@@ -877,9 +873,7 @@ async def _post_if_available(
async def _profile_cmd(cmd: str, payload: dict, e_url: str, p_url: str, d_url: str):
"""
Fire & forget to both clusters, tolerate 404.
"""
"""Fire & forget to both clusters, tolerate 404."""
headers = {"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY', '')}"}
encode_task = _post_if_available(
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file provides a disaggregated prefilling proxy demo to demonstrate an
"""This file provides a disaggregated prefilling proxy demo to demonstrate an
example usage of XpYd disaggregated prefilling.
We can launch multiple vllm instances (2 for prefill and 2 for decode), and
launch this proxy demo through:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer
"""Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer.
This proxy sits between clients and a vLLM Prefill/Decode (P/D) deployment,
routing multi-turn chat requests so that each turn reuses KV cache blocks
@@ -54,7 +53,7 @@ Benchmarking:
it the benchmark sends OpenAI-schema-compliant payloads and every
turn lands as a cache MISS in this proxy.
Example:
Example:
python benchmarks/multi_turn/benchmark_serving_multi_turn.py \\
--model <MODEL> --served-model-name <NAME> \\
--url http://<proxy_host>:8000 \\
@@ -67,6 +66,7 @@ Benchmarking:
Dependencies:
pip install fastapi uvicorn httpx
"""
from __future__ import annotations
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Push-mode disaggregated prefilling proxy demo.
"""Push-mode disaggregated prefilling proxy demo.
Companion to ``disagg_proxy_demo.py`` (pull mode). The client-facing API is
the same; the difference is in how P and D coordinate the KV transfer:
@@ -6,7 +6,7 @@ from vllm.config import KVTransferConfig
def read_prompts():
"""Read prompts from output.txt"""
"""Read prompts from output.txt."""
prompts = []
try:
with open("output.txt") as f:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use FlexKV with vLLM for prefix caching.
"""This example shows how to use FlexKV with vLLM for prefix caching.
FlexKV is a distributed KV Store and multi-level cache management system for
ultra-large-scale LLM inference.
@@ -7,7 +7,7 @@ from vllm.config import KVTransferConfig
def read_prompts():
"""Read prompts from prefill_output.txt"""
"""Read prompts from prefill_output.txt."""
prompts = []
try:
with open("prefill_output.txt") as f:
@@ -119,8 +119,7 @@ class LoadRecoveryExampleConnector(ExampleConnector):
def update_state_after_alloc(
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""Update KVConnector state after block allocation.
If blocks were allocated, add to _requests_need_load,
such that we load the KVs in the next forward pass.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file demonstrates the example usage of CPU offloading
"""This file demonstrates the example usage of CPU offloading
with LMCache in vLLM v1.
Note that `lmcache` is needed to run this example.
@@ -14,9 +14,7 @@ from fastapi.responses import StreamingResponse
@asynccontextmanager
async def lifespan(app: FastAPI):
"""
Lifespan context manager to handle startup and shutdown events.
"""
"""Lifespan context manager to handle startup and shutdown events."""
# Startup: Initialize clients
prefiller_base_url = (
f"http://{global_args.prefiller_host}:{global_args.prefiller_port}/v1"
@@ -106,9 +104,7 @@ app.state.decode_client = None
async def send_request_to_service(
client: httpx.AsyncClient, endpoint: str, req_data: dict
):
"""
Send a request to a service using a persistent client.
"""
"""Send a request to a service using a persistent client."""
req_data = req_data.copy()
req_data["max_tokens"] = 1
if "max_completion_tokens" in req_data:
@@ -128,9 +124,7 @@ async def send_request_to_service(
async def stream_service_response(
client: httpx.AsyncClient, endpoint: str, req_data: dict
):
"""
Asynchronously stream the response from a service using a persistent client.
"""
"""Asynchronously stream the response from a service using a persistent client."""
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
async with client.stream(
"POST", endpoint, json=req_data, headers=headers
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file demonstrates the example usage of remote KV cache sharing
"""This file demonstrates the example usage of remote KV cache sharing
with LMCache.
We will launch 2 vllm instances, and launch an additional LMCache server.
KV cache is transferred in the following manner:
@@ -65,9 +65,7 @@ async def get_prefiller_info(prefill_clients: list, ready: asyncio.Event):
@asynccontextmanager
async def lifespan(app: FastAPI):
"""
Lifespan context manager to handle startup and shutdown events.
"""
"""Lifespan context manager to handle startup and shutdown events."""
# Startup: Initialize client pools for prefiller and decoder services
app.state.prefill_clients = []
app.state.decode_clients = []
@@ -180,10 +178,12 @@ def _parse_prefill_urls(prefill_list):
"""Parse prefill URLs from --prefill arguments.
Format: --prefill URL [BOOTSTRAP_PORT]
Example:
--prefill http://prefill1:8080 9000 # With bootstrap port
--prefill http://prefill2:8080 none # Explicitly no bootstrap port
--prefill http://prefill3:8080 # Defaults to no bootstrap port
"""
if not prefill_list:
return []
@@ -228,8 +228,7 @@ def _parse_decode_urls(decode_list):
def get_next_client(app, service_type: str):
"""
Get the next client in round-robin fashion.
"""Get the next client in round-robin fashion.
Args:
app: The FastAPI app instance
@@ -237,6 +236,7 @@ def get_next_client(app, service_type: str):
Returns:
The next client to use
"""
if service_type == "prefill":
return next(app.state.prefill_iterator)
@@ -250,9 +250,7 @@ def get_next_client(app, service_type: str):
async def send_request_to_service(
client_info: dict, dp_rank: int, endpoint: str, req_data: dict, request_id: str
):
"""
Send a request to a service using a client from the pool.
"""
"""Send a request to a service using a client from the pool."""
req_data = req_data.copy()
req_data["kv_transfer_params"] = {
"do_remote_decode": True,
@@ -288,9 +286,7 @@ async def stream_service_response(
req_data: dict,
request_id: str,
):
"""
Asynchronously stream response from a service using a client from the pool.
"""
"""Asynchronously stream response from a service using a client from the pool."""
headers = {
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
"X-Request-Id": request_id,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstration script for Automatic Prefix Caching (APC) in vLLM.
"""Demonstration script for Automatic Prefix Caching (APC) in vLLM.
Automatic Prefix Caching (APC) allows the vLLM engine to reuse cached
KV (key-value) pairs from previous prompts if a new query shares the same
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstrates how to achieve reproducibility in vLLM.
"""Demonstrates how to achieve reproducibility in vLLM.
Main article: https://docs.vllm.ai/en/latest/usage/reproducibility.html
"""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This script demonstrates how to extend the context length
"""This script demonstrates how to extend the context length
of a Qwen model using the YARN method (rope_parameters)
and run a simple chat example.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Usage:
"""Usage:
Single node:
python examples/features/data_parallel/data_parallel_offline.py \
--model="ibm-research/PowerMoE-3b" \
@@ -18,7 +18,7 @@ class EventBatch(msgspec.Struct, array_like=True, omit_defaults=True, gc=False):
class KVCacheEvent(msgspec.Struct, omit_defaults=True, gc=False, tag=True):
"""Base class for all KV cache-related events"""
"""Base class for all KV cache-related events."""
class BlockStored(KVCacheEvent):
+1 -1
View File
@@ -48,7 +48,7 @@ from vllm.v1.sample.logits_processor.builtin import process_dict_updates
# Hypothetical custom logits processor
class DummyLogitsProcessor(LogitsProcessor):
"""Fake logit processor to support unit testing and examples"""
"""Fake logit processor to support unit testing and examples."""
@classmethod
def validate_params(cls, params: SamplingParams):
@@ -55,10 +55,11 @@ logger = init_logger(__name__)
class DummyPerReqLogitsProcessor:
"""The request-level logits processor masks out all logits except the
token id identified by `target_token`"""
token id identified by `target_token`
"""
def __init__(self, target_token: int) -> None:
"""Specify `target_token`"""
"""Specify `target_token`."""
self.target_token = target_token
def __call__(
@@ -74,7 +75,8 @@ class DummyPerReqLogitsProcessor:
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
"""Example of wrapping a fake request-level logit processor to create a
batch-level logits processor"""
batch-level logits processor
"""
@classmethod
def validate_params(cls, params: SamplingParams):
@@ -103,6 +105,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
Returns:
`Callable` request logits processor, or None
"""
target_token: Any | None = params.extra_args and params.extra_args.get(
"target_token"
@@ -56,10 +56,11 @@ logger = init_logger(__name__)
class DummyPerReqLogitsProcessor:
"""The request-level logits processor masks out all logits except the
token id identified by `target_token`"""
token id identified by `target_token`
"""
def __init__(self, target_token: int) -> None:
"""Specify `target_token`"""
"""Specify `target_token`."""
self.target_token = target_token
def __call__(
@@ -75,7 +76,8 @@ class DummyPerReqLogitsProcessor:
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
"""Example of overriding the wrapper class `__init__()` in order to utilize
info about the device type"""
info about the device type
"""
@classmethod
def validate_params(cls, params: SamplingParams):
@@ -111,6 +113,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
Returns:
`Callable` request logits processor, or None
"""
if (
not self.is_cuda
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use LoRA with different quantization techniques
"""This example shows how to use LoRA with different quantization techniques
for offline inference.
Requires HuggingFace credentials for access.
@@ -73,7 +72,6 @@ def initialize_engine(
model: str, quantization: str, lora_repo: str | None
) -> LLMEngine:
"""Initialize the LLMEngine."""
engine_args = EngineArgs(
model=model,
quantization=quantization,
@@ -86,7 +84,6 @@ def initialize_engine(
def main():
"""Main function that sets up and runs the prompt processing."""
test_configs = [
# QLoRA (https://arxiv.org/abs/2305.14314)
{
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use the multi-LoRA functionality
"""This example shows how to use the multi-LoRA functionality
for offline inference.
Requires HuggingFace credentials for access to Llama2.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test pause/resume with Data Parallel (DP) via HTTP API.
"""Test pause/resume with Data Parallel (DP) via HTTP API.
This example demonstrates coordinated pause/resume across multiple DP ranks.
The pause synchronizes across all DP engines via all-reduce.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstrates how to generate prompt embeddings using
"""Demonstrates how to generate prompt embeddings using
Hugging Face Transformers and use them as input to vLLM
for both single and batch inference.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file demonstrates preempt requests when using the `LLMEngine`
"""This file demonstrates preempt requests when using the `LLMEngine`
for processing prompts with various sampling parameters.
"""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Validates the loading of a model saved with the sharded_state format.
"""Validates the loading of a model saved with the sharded_state format.
This script demonstrates how to load a model that was previously saved
using save_sharded_state_offline.py and validates it by running inference.
Example usage:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Saves each worker's model state dict directly to a checkpoint, which enables a
"""Saves each worker's model state dict directly to a checkpoint, which enables a
fast load path for large tensor-parallel models where each worker only needs to
read its own shard rather than the entire checkpoint.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file demonstrates the usage of text generation with an LLM model,
"""This file demonstrates the usage of text generation with an LLM model,
comparing the performance with and without speculative decoding.
Note that this example is out of date and not supported in vLLM v1.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file demonstrates the example usage of structured outputs
"""This file demonstrates the example usage of structured outputs
in vLLM. It shows how to apply different constraints such as choice,
regex, json schema, and grammar to produce structured and formatted
results based on specific prompts.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
experimental support for data-parallel inference with torchrun
"""experimental support for data-parallel inference with torchrun
Note the data load balancing and distribution is done out of the vllm engine,
no internal lb supported in external_launcher mode.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
experimental support for tensor-parallel inference with torchrun,
"""experimental support for tensor-parallel inference with torchrun,
see https://github.com/vllm-project/vllm/issues/11400 for
the motivation and use case for this example.
run the script with `torchrun --nproc-per-node=4 torchrun_example_offline.py`,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference
"""This example shows how to use vLLM for running offline inference
with the correct prompt format on audio language models.
For most models, the prompt format should follow corresponding examples
@@ -286,8 +285,7 @@ def run_minicpmo(question: str, audio_count: int) -> ModelRequestData:
# Phi-4-multimodal-instruct
def run_phi4mm(question: str, audio_count: int) -> ModelRequestData:
"""
Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
"""Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
show how to process audio inputs.
"""
model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference with
"""This example shows how to use vLLM for running offline inference with
the explicit/implicit prompt format on enc-dec LMMs for text generation.
"""
@@ -56,8 +55,7 @@ def run_whisper():
def run_fireredasr2():
"""
FireRedASR2 – Automatic Speech Recognition model.
"""FireRedASR2 – Automatic Speech Recognition model.
This model uses a Conformer encoder + Qwen2 LLM decoder architecture
for speech-to-text transcription. Audio is passed via the implicit
@@ -43,7 +43,6 @@ headers = {"User-Agent": "vLLM Example Client"}
def encode_base64_content_from_url(content_url: str) -> str:
"""Encode a content retrieved from a remote url to base64 format."""
with requests.get(content_url, headers=headers) as response:
response.raise_for_status()
result = base64.b64encode(response.content).decode("utf-8")
@@ -53,7 +52,6 @@ def encode_base64_content_from_url(content_url: str) -> str:
def encode_base64_content_from_file(file_path: str) -> str:
"""Encode a local file content to base64 format."""
with open(file_path, "rb") as file:
file_content = file.read()
result = base64.b64encode(file_content).decode("utf-8")
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference
"""This example shows how to use vLLM for running offline inference
with the correct prompt format on Qwen2.5-Omni (thinker only).
"""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference
"""This example shows how to use vLLM for running offline inference
with the correct prompt format on Qwen3-Omni (thinker only).
"""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference with
"""This example shows how to use vLLM for running offline inference with
multi-image input on vision language models for text generation,
using the chat template defined by the model.
"""
@@ -913,11 +912,9 @@ def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData:
def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
"""
Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
"""Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
show how to process multi images inputs.
"""
model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
# Since the vision-lora and speech-lora co-exist with the base model,
# we have to manually specify the path of the lora weights.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference with
"""This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.
For most models, the prompt format should follow corresponding examples
@@ -1660,8 +1659,7 @@ def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
# Phi-4-multimodal-instruct
def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
"""
Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
"""Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
show how to process image inputs.
"""
assert modality == "image"
@@ -2283,8 +2281,7 @@ MODELS_SUPPORT_VIT_CUDA_GRAPH = [
def get_multi_modal_input(args):
"""
return {
"""Return {
"data": image or video,
"question": question,
}
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Trace-replay with vLLM offline inference.
"""Trace-replay with vLLM offline inference.
Trace-replay lets you supply a known sequence of decode token IDs alongside
the prompt. Instead of sampling from the model distribution, the engine
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example script demonstrating long text embedding with chunked processing in vLLM.
"""Example script demonstrating long text embedding with chunked processing in vLLM.
This example shows how to use vLLM's chunked processing feature to handle text
inputs that exceed the model's maximum token length. The feature automatically
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
This example shows how to use vLLM for running offline inference with
"""This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for multimodal embedding.
For most models, the prompt format should follow corresponding examples
@@ -37,8 +37,7 @@ def create_chat_embeddings(
add_special_tokens: bool = False,
padding: str | NotGiven = NOT_GIVEN,
) -> CreateEmbeddingResponse:
"""
Convenience function for accessing vLLM's Chat Embeddings API,
"""Convenience function for accessing vLLM's Chat Embeddings API,
which is an extension of OpenAI's existing Embeddings API.
"""
return client.post(
@@ -56,13 +55,11 @@ def create_chat_embeddings(
def run_clip(client: OpenAI, model: str):
"""
Start the server using:
"""Start the server using:
vllm serve openai/clip-vit-base-patch32 \
--runner pooling
"""
response = create_chat_embeddings(
client,
messages=[
@@ -97,8 +94,7 @@ def run_clip(client: OpenAI, model: str):
def run_dse_qwen2_vl(client: OpenAI, model: str):
"""
Start the server using:
"""Start the server using:
vllm serve MrLight/dse-qwen2-2b-mrl-v1 \
--runner pooling \
@@ -159,14 +155,12 @@ def run_dse_qwen2_vl(client: OpenAI, model: str):
def run_qwen3_vl(client: OpenAI, model: str):
"""
Start the server using:
"""Start the server using:
vllm serve Qwen/Qwen3-VL-Embedding-2B \
--runner pooling \
--max-model-len 8192
"""
default_instruction = "Represent the user's input."
print("Text embedding output:")
@@ -266,14 +260,12 @@ def run_qwen3_vl(client: OpenAI, model: str):
def run_siglip(client: OpenAI, model: str):
"""
Start the server using:
"""Start the server using:
vllm serve google/siglip-base-patch16-224 \
--runner pooling \
--chat-template template_basic.jinja
"""
response = create_chat_embeddings(
client,
messages=[
@@ -311,8 +303,7 @@ def run_siglip(client: OpenAI, model: str):
def run_vlm2vec(client: OpenAI, model: str):
"""
Start the server using:
"""Start the server using:
vllm serve TIGER-Lab/VLM2Vec-Full \
--runner pooling \
@@ -320,7 +311,6 @@ def run_vlm2vec(client: OpenAI, model: str):
--max-model-len 4096 \
--chat-template examples/pooling/embed/template/vlm2vec_phi3v.jinja
"""
response = create_chat_embeddings(
client,
messages=[
@@ -85,8 +85,7 @@ def generate_datamodule():
def process_channel_group(orig_img, channels):
"""
Args:
"""Args:
orig_img: torch.Tensor representing original image (reference)
with shape = (bands, H, W).
channels: list of indices representing RGB channels.
@@ -94,8 +93,8 @@ def process_channel_group(orig_img, channels):
Returns:
torch.Tensor with shape (num_channels, height, width)
for original image
"""
"""
orig_img = orig_img[channels, ...]
valid_mask = torch.ones_like(orig_img, dtype=torch.bool)
valid_mask[orig_img == NO_DATA_FLOAT] = False
@@ -121,8 +120,8 @@ def read_geotiff(file_path: str):
Returns:
np.ndarray with shape (bands, height, width)
meta info dict
"""
"""
with rasterio.open(file_path) as src:
img = src.read()
meta = src.meta
@@ -142,8 +141,8 @@ def save_geotiff(image, output_path: str, meta: dict):
image: np.ndarray with shape (bands, height, width)
output_path: path where to save the image
meta: dict with meta info.
"""
"""
with rasterio.open(output_path, "w", **meta) as dest:
for i in range(image.shape[0]):
dest.write(image[i, :, :], i + 1)
@@ -172,12 +171,14 @@ def load_example(
images in *file_paths*.
std: list containing std values for each band in the
images in *file_paths*.
indices: optional band indices to select from each image. All bands
are kept when None.
Returns:
np.array containing created example
list of meta info for each image in *file_paths*
"""
"""
imgs = []
metas = []
temporal_coords = []
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example offline usage of sequence reward models.
"""Example offline usage of sequence reward models.
The key distinction between sequence classification and token classification
lies in their output granularity: sequence classification produces a single
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example online usage of sequence reward models.
"""Example online usage of sequence reward models.
Run `vllm serve <model> --runner pooling`
to start up the server in vLLM. e.g.
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example offline usage of token reward models.
"""Example offline usage of token reward models.
The key distinction between sequence classification and token classification
lies in their output granularity: sequence classification produces a single
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example online usage of token reward models.
"""Example online usage of token reward models.
Run `vllm serve <model> --runner pooling`
to start up the server in vLLM. e.g.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example of using the OpenAI entrypoint's rerank API which is compatible with
"""Example of using the OpenAI entrypoint's rerank API which is compatible with
the Cohere SDK: https://github.com/cohere-ai/cohere-python
Note that `pip install cohere` is needed to run this example.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example of using ColBERT late interaction models for reranking and scoring.
"""Example of using ColBERT late interaction models for reranking and scoring.
ColBERT (Contextualized Late Interaction over BERT) uses per-token embeddings
and MaxSim scoring for document reranking, providing better accuracy than
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example of using ColModernVBERT late interaction model for reranking.
"""Example of using ColModernVBERT late interaction model for reranking.
ColModernVBERT is a multi-modal ColBERT-style model combining a SigLIP
vision encoder with a ModernBERT text encoder. It produces per-token
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example of using ColQwen3.5 late interaction model for reranking.
"""Example of using ColQwen3.5 late interaction model for reranking.
ColQwen3.5 is a multi-modal ColBERT-style model based on Qwen3.5.
It produces per-token embeddings and uses MaxSim scoring for retrieval
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Example of using ColQwen3 late interaction model for reranking and scoring.
"""Example of using ColQwen3 late interaction model for reranking and scoring.
ColQwen3 is a multi-modal ColBERT-style model based on Qwen3-VL.
It produces per-token embeddings and uses MaxSim scoring for retrieval
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Script to convert Large Language Models (LLMs) to Sequence Classification models.
"""Script to convert Large Language Models (LLMs) to Sequence Classification models.
This is particularly useful for converting reranker models that use next-token
prediction to a sequence classification format for compatibility with standard
classification and rerank pipelines.
@@ -35,8 +34,7 @@ import transformers
def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device):
"""
This method extracts the difference between weights for 'true' and 'false' tokens
"""This method extracts the difference between weights for 'true' and 'false' tokens
from the language model head to create a single classification weight vector.
Args:
@@ -47,6 +45,7 @@ def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device):
device: Target device (cpu/cuda)
Reference: https://huggingface.co/Qwen/Qwen3-Reranker-0.6B/discussions/3
"""
assert len(tokens) == 2, (
"Method requires exactly two tokens for binary classification"
@@ -73,8 +72,7 @@ def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device):
def no_post_processing(causal_lm, seq_cls_model, tokenizer, tokens, device):
"""
Directly use token weights from the language model head for classification.
"""Directly use token weights from the language model head for classification.
This method maps each classification label directly to a corresponding token
in the vocabulary without additional transformation.
@@ -85,6 +83,7 @@ def no_post_processing(causal_lm, seq_cls_model, tokenizer, tokens, device):
tokenizer: Model tokenizer
tokens: List of tokens representing class labels
device: Target device (cpu/cuda)
"""
# Get the language model head weights (vocabulary_size x hidden_size)
lm_head_weights = causal_lm.lm_head.weight
@@ -110,8 +109,7 @@ method_map = {
def converting(
model_name, classifier_from_tokens, path, method, use_sep_token=False, device="cpu"
):
"""
Main conversion function to transform a CausalLM model to SequenceClassification.
"""Main conversion function to transform a CausalLM model to SequenceClassification.
Args:
model_name: Name or path of the pretrained model
@@ -120,6 +118,7 @@ def converting(
method: Conversion method ('from_2_way_softmax' or 'no_post_processing')
use_sep_token: Whether to use separating token in the sequence classification model
device: Device to load the model on ('cpu' or 'cuda')
"""
assert method in method_map, f"Unknown method: {method}"
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
What is the difference between the official original version and one
"""What is the difference between the official original version and one
that has been converted into a sequence classification model?
Qwen3-Reranker is a language model that doing reranker by using the
@@ -34,8 +33,7 @@ model_name = "Qwen/Qwen3-Reranker-0.6B"
def get_llm() -> LLM:
"""
Initializes and returns the LLM model for Qwen3-Reranker.
"""Initializes and returns the LLM model for Qwen3-Reranker.
Returns:
LLM: Configured vLLM instance for reranking tasks.
@@ -43,6 +41,7 @@ def get_llm() -> LLM:
Note:
This function loads the ORIGINAL Qwen3-Reranker model with specific
overrides to make it compatible with vLLM's score API.
"""
return LLM(
# Specify the original model from HuggingFace
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
What is the difference between the official original version and one
"""What is the difference between the official original version and one
that has been converted into a sequence classification model?
Qwen3-Reranker is a language model that doing reranker by using the
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example of using the OpenAI entrypoint's rerank API which is compatible with
"""Example of using the OpenAI entrypoint's rerank API which is compatible with
Jina and Cohere https://jina.ai/reranker
run: vllm serve BAAI/bge-reranker-base
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example online usage of Score API.
"""Example online usage of Score API.
Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
"""
@@ -68,13 +68,13 @@ def get_chat_template(model: str) -> str:
def get_hf_overrides(model: str) -> dict[str, Any]:
"""Convert Large Language Models (LLMs) to Sequence Classification models.
note:
Note:
Some reranking models require special configuration overrides to work
correctly with vLLM's score API.
Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_offline.py
Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/convert_model_to_seq_cls.py
"""
"""
model_name_to_hf_overrides_map = {
"BAAI/bge-reranker-v2-gemma": {
"architectures": ["GemmaForSequenceClassification"],
@@ -122,7 +122,6 @@ def get_hf_overrides(model: str) -> dict[str, Any]:
def main(args: Namespace):
"""Main execution function for the reranking example."""
# Get the overrides for the specified model
args.hf_overrides = get_hf_overrides(args.model)
@@ -1,15 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Example of using the rerank API with template.
"""Example of using the rerank API with template.
This script demonstrates how to interact with a vLLM server running
a reranking model via the REST API.
Before running this script, start the vLLM server with one of the
supported reranking models using the commands below.
note:
Note:
Some reranking models require special configuration overrides to work correctly
with vLLM's score API.
Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_online.py
@@ -21,6 +20,7 @@ run:
vllm serve mixedbread-ai/mxbai-rerank-base-v2 --hf_overrides '{"architectures": ["Qwen2ForSequenceClassification"],"classifier_from_token": ["0", "1"], "method": "from_2_way_softmax"}' --chat-template examples/pooling/score/template/mxbai_rerank_v2.jinja
vllm serve nvidia/llama-nemotron-rerank-1b-v2 --runner pooling --trust-remote-code --chat-template examples/pooling/score/template/nemotron-rerank.jinja
vllm serve Qwen/Qwen3-Reranker-0.6B --runner pooling --hf_overrides '{"architectures": ["Qwen3ForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}' --chat-template examples/pooling/score/template/qwen3_reranker.jinja
"""
import json
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Example Python client for multimodal rerank API which is compatible with
"""Example Python client for multimodal rerank API which is compatible with
Jina and Cohere https://jina.ai/reranker
Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use vLLM for running offline inference with
"""This example shows how to use vLLM for running offline inference with
vision language reranker models for multimodal scoring tasks.
Vision language rerankers score the relevance between a text query and
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Example online usage of Score API.
"""Example online usage of Score API.
Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
e.g.
@@ -3,8 +3,7 @@
# Adapted from Qwen3-ForcedAligner inference:
# https://github.com/QwenLM/Qwen3-ASR
"""
Offline forced alignment example using Qwen3-ForcedAligner-0.6B.
"""Offline forced alignment example using Qwen3-ForcedAligner-0.6B.
Forced alignment takes audio and reference text as input and produces
word-level timestamps. The model predicts a time bin at each <timestamp>
@@ -3,8 +3,7 @@
# Adapted from Qwen3-ForcedAligner inference:
# https://github.com/QwenLM/Qwen3-ASR
"""
Online forced alignment example using Qwen3-ForcedAligner-0.6B-hf.
"""Online forced alignment example using Qwen3-ForcedAligner-0.6B-hf.
Forced alignment takes audio and reference text as input and produces
word-level timestamps. The model predicts a time bin at each <timestamp>
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://huggingface.co/boltuix/NeuroBERT-NER
"""
Example online usage of Pooling API for Named Entity Recognition (NER).
"""Example online usage of Pooling API for Named Entity Recognition (NER).
Run `vllm serve <model> --runner pooling`
to start up the server in vLLM. e.g.
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Example online usage of Pooling API for ColQwen3 multi-vector retrieval.
"""Example online usage of Pooling API for ColQwen3 multi-vector retrieval.
ColQwen3 is a multi-modal late interaction model based on Qwen3-VL that
produces per-token embeddings (320-dim, L2-normalized) for both text and
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Example online usage of the Jina Reranker v3 score and rerank APIs with a task
"""Example online usage of the Jina Reranker v3 score and rerank APIs with a task
instruction.
Run `vllm serve jinaai/jina-reranker-v3 --runner pooling` to start up the
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example online usage of Pooling API for multi vector retrieval.
"""Example online usage of Pooling API for multi vector retrieval.
Run `vllm serve <model> --runner pooling`
to start up the server in vLLM. e.g.
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This example shows how to use Ray Data for data parallel batch inference.
"""This example shows how to use Ray Data for data parallel batch inference.
Ray Data is a data processing framework that can process very large datasets
with first-class support for vLLM.
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Deploy DeepSeek R1 or V3 with Ray Serve LLM.
"""Deploy DeepSeek R1 or V3 with Ray Serve LLM.
Ray Serve LLM is a scalable and production-grade model serving library built
on the Ray distributed computing framework and first-class support for the vLLM engine.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
An example demonstrates how to use tool calling with reasoning models
"""An example demonstrates how to use tool calling with reasoning models
like QwQ-32B. The reasoning will not be parsed by the tool
calling process; only the final output will be parsed.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
An example shows how to generate chat completions from reasoning models
"""An example shows how to generate chat completions from reasoning models
like DeepSeekR1.
To run this example, you need to start the vLLM server
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
An example shows how to generate chat completions from reasoning models
"""An example shows how to generate chat completions from reasoning models
like DeepSeekR1.
To run this example, you need to start the vLLM server with the reasoning
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Set up this example by starting a vLLM OpenAI-compatible server.
"""Set up this example by starting a vLLM OpenAI-compatible server.
Reasoning models can be used through the Responses API as seen here
https://platform.openai.com/docs/api-reference/responses
For example:
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstrates async reinforcement learning using vLLM and Ray,
"""Demonstrates async reinforcement learning using vLLM and Ray,
with native weight syncing APIs and batch-invariant generation.
The script separates training and inference workloads onto distinct GPUs
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the
"""RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the
data plane.
* OpenAI-compatible API for inference requests
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the
"""RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the
data plane.
* OpenAI-compatible API for inference requests
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC**
"""RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC**
weight transfer and **packed** tensors.
Multi-rank version of `rlhf_http_ipc.py`: the trainer is 4 FSDP2 Ray actors
+2 -4
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs).
"""RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs).
8-GPU layout:
Training — 4 GPUs, PyTorch FSDP2 (fully_shard), as Ray actors
@@ -74,8 +73,7 @@ BASE_URL = f"http://localhost:{SERVER_PORT}"
@ray.remote(num_gpus=1)
class FSDPTrainWorker:
"""
One FSDP2 training worker per GPU. Four of these form the FSDP group.
"""One FSDP2 training worker per GPU. Four of these form the FSDP group.
Rank 0 additionally drives weight transfer to the vLLM server.
"""
+2 -1
View File
@@ -101,7 +101,8 @@ PROMPTS = ["The capital of France is", "The future of AI is"]
class FSDPTrainWorker:
"""One FSDP2 rank per GPU. Identical to the 8-GPU example's worker: the
sharded-RDT engine owns the NIXL serve surface, so this stays a plain Ray
actor with no producer mixin."""
actor with no producer mixin.
"""
def __init__(self, model_name, rank, world_size, master_addr, master_port):
self.rank = rank
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end example for routed experts capture with hybrid models.
"""End-to-end example for routed experts capture with hybrid models.
Validates that:
1. routed_experts is returned in CompletionOutput for MoE models.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This script demonstrates how to use the vLLM API server to perform audio
"""This script demonstrates how to use the vLLM API server to perform audio
transcription with the `openai/whisper-large-v3` model.
Before running this script, you must start the vLLM server with the following command:
@@ -35,8 +34,7 @@ def sync_openai(
hotwords: str = None,
prompt: str | None = None,
):
"""
Perform synchronous transcription using OpenAI-compatible API.
"""Perform synchronous transcription using OpenAI-compatible API.
The optional ``prompt`` is the OpenAI-API ``prompt`` field (style /
vocabulary hint). It is wired through model-by-model: Whisper uses it
@@ -69,9 +67,7 @@ async def stream_openai_response(
hotwords: str = None,
prompt: str | None = None,
):
"""
Perform asynchronous transcription using OpenAI-compatible API.
"""
"""Perform asynchronous transcription using OpenAI-compatible API."""
print("\ntranscription result [stream]:", end=" ")
with open(audio_path, "rb") as f:
transcription = await client.audio.transcriptions.create(
@@ -98,9 +94,7 @@ async def stream_openai_response(
def stream_api_response(audio_path: str, model: str, openai_api_base: str):
"""
Perform streaming transcription using raw HTTP requests to the vLLM API server.
"""
"""Stream a transcription using raw HTTP requests to the vLLM API server."""
import json
import os
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This script demonstrates how to use the vLLM Realtime WebSocket API to perform
"""This script demonstrates how to use the vLLM Realtime WebSocket API to perform
audio transcription by uploading an audio file.
Before running this script, you must start the vLLM server with a realtime-capable
@@ -34,9 +33,7 @@ from vllm.multimodal.media.audio import load_audio
def audio_to_pcm16_base64(audio_path: str) -> str:
"""
Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
"""
"""Load an audio file and convert it to base64-encoded PCM16 @ 16kHz."""
# Load audio and resample to 16kHz mono
audio, _ = load_audio(audio_path, sr=16000, mono=True)
# Convert to PCM16
@@ -46,9 +43,7 @@ def audio_to_pcm16_base64(audio_path: str) -> str:
async def realtime_transcribe(audio_path: str, host: str, port: int, model: str):
"""
Connect to the Realtime API and transcribe an audio file.
"""
"""Connect to the Realtime API and transcribe an audio file."""
uri = f"ws://{host}:{port}/v1/realtime"
async with websockets.connect(uri) as ws:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API.
"""Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API.
Start the vLLM server first:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Set up this example by starting a vLLM OpenAI-compatible server with tool call
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
options enabled. For example:
IMPORTANT: for mistral, you must use one of the provided mistral tool call
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
To run this example, you can start the vLLM server
"""To run this example, you can start the vLLM server
without any specific flags:
```bash
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Set up this example by starting a vLLM OpenAI-compatible server with tool call
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
options enabled for xLAM-2 models:
vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
@@ -105,8 +104,7 @@ tool_functions = {
def process_response(response, tool_functions, original_query):
"""Process a non-streaming response with possible tool calls"""
"""Process a non-streaming response with possible tool calls."""
print("\n--- Response Output ---")
# Check if the response has content
@@ -195,7 +193,7 @@ def process_response(response, tool_functions, original_query):
def run_test_case(query, test_name):
"""Run a single test case with the given query"""
"""Run a single test case with the given query."""
print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}")
print(f"Query: '{query}'")
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
Set up this example by starting a vLLM OpenAI-compatible server with tool call
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
options enabled for xLAM-2 models:
vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
@@ -107,7 +106,7 @@ tool_functions = {
def process_stream(response, tool_functions, original_query):
"""Process a streaming response with possible tool calls"""
"""Process a streaming response with possible tool calls."""
# Track multiple tool calls
tool_calls = {} # Dictionary to store tool calls by ID
@@ -223,7 +222,7 @@ def process_stream(response, tool_functions, original_query):
def run_test_case(query, test_name):
"""Run a single test case with the given query"""
"""Run a single test case with the given query."""
print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}")
print(f"Query: '{query}'")
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example demonstrating MCP (Model Context Protocol) tools with the Responses API.
"""Example demonstrating MCP (Model Context Protocol) tools with the Responses API.
This example shows how to use MCP tools with different allowed_tools configurations:
1. No filter (allows all tools from the MCP server)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Set up this example by starting a vLLM OpenAI-compatible server with tool call
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
options enabled.
Reasoning models can be used through the Responses API as seen here
https://platform.openai.com/docs/api-reference/responses
@@ -17,8 +16,7 @@ from openai import OpenAI
def get_weather(latitude: float, longitude: float) -> str:
"""
Mock function to simulate getting weather data.
"""Mock function to simulate getting weather data.
In a real application, this would call an external weather API.
"""
return f"Current temperature at ({latitude}, {longitude}) is 20°C."