Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
NOTE: This API server is used only for demonstrating usage of AsyncEngine
|
||||
"""NOTE: This API server is used only for demonstrating usage of AsyncEngine
|
||||
and simple performance benchmarks. It is not intended for production use.
|
||||
For production use, we recommend using our OpenAI compatible server.
|
||||
We are also not going to accept PRs modifying this file.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
vLLM Chat Assistant - A Streamlit Web Interface
|
||||
"""vLLM Chat Assistant - A Streamlit Web Interface.
|
||||
|
||||
A streamlined chat interface that quickly integrates
|
||||
with vLLM API server.
|
||||
@@ -78,6 +77,7 @@ def create_new_chat_session():
|
||||
- current_session: Sets to new session ID
|
||||
- active_session: Sets to new session ID
|
||||
- messages: Resets to empty list
|
||||
|
||||
"""
|
||||
session_id = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
st.session_state.sessions[session_id] = []
|
||||
@@ -101,6 +101,7 @@ def switch_to_chat_session(session_id):
|
||||
- current_session: Updated to specified session_id
|
||||
- active_session: Updated to specified session_id
|
||||
- messages: Loaded from sessions[session_id]
|
||||
|
||||
"""
|
||||
st.session_state.current_session = session_id
|
||||
st.session_state.active_session = session_id
|
||||
@@ -134,6 +135,7 @@ def get_llm_response(messages, model, reason, content_ph=None, reasoning_ph=None
|
||||
Note:
|
||||
The function uses streamlit placeholders for live updates.
|
||||
When reason=True, the reasoning process appears above the response.
|
||||
|
||||
"""
|
||||
full_text = ""
|
||||
think_text = ""
|
||||
@@ -256,6 +258,7 @@ def server_supports_reasoning():
|
||||
|
||||
Returns:
|
||||
bool: True if the model supports reasoning, False otherwise
|
||||
|
||||
"""
|
||||
resp = client.chat.completions.create(
|
||||
model=model,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Retrieval Augmented Generation (RAG) Implementation with Langchain
|
||||
"""Retrieval Augmented Generation (RAG) Implementation with Langchain
|
||||
==================================================================
|
||||
|
||||
This script demonstrates a RAG implementation using LangChain, Milvus
|
||||
@@ -35,6 +34,7 @@ Notes:
|
||||
- Ensure both vLLM services are running before executing
|
||||
- Default ports: 8000 (embedding), 8001 (chat)
|
||||
- First run may take time to download models
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -52,9 +52,7 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
|
||||
|
||||
def load_and_split_documents(config: dict[str, Any]):
|
||||
"""
|
||||
Load and split documents from web URL
|
||||
"""
|
||||
"""Load and split documents from web URL."""
|
||||
try:
|
||||
loader = WebBaseLoader(web_paths=(config["url"],))
|
||||
docs = loader.load()
|
||||
@@ -70,9 +68,7 @@ def load_and_split_documents(config: dict[str, Any]):
|
||||
|
||||
|
||||
def init_vectorstore(config: dict[str, Any], documents: list[Document]):
|
||||
"""
|
||||
Initialize vector store with documents
|
||||
"""
|
||||
"""Initialize vector store with documents."""
|
||||
return Milvus.from_documents(
|
||||
documents=documents,
|
||||
embedding=OpenAIEmbeddings(
|
||||
@@ -86,9 +82,7 @@ def init_vectorstore(config: dict[str, Any], documents: list[Document]):
|
||||
|
||||
|
||||
def init_llm(config: dict[str, Any]):
|
||||
"""
|
||||
Initialize llm
|
||||
"""
|
||||
"""Initialize llm."""
|
||||
return ChatOpenAI(
|
||||
model=config["chat_model"],
|
||||
openai_api_key=config["vllm_api_key"],
|
||||
@@ -97,9 +91,7 @@ def init_llm(config: dict[str, Any]):
|
||||
|
||||
|
||||
def get_qa_prompt():
|
||||
"""
|
||||
Get question answering prompt template
|
||||
"""
|
||||
"""Get question answering prompt template."""
|
||||
template = """You are an assistant for question-answering tasks.
|
||||
Use the following pieces of retrieved context to answer the question.
|
||||
If you don't know the answer, just say that you don't know.
|
||||
@@ -112,16 +104,12 @@ Answer:
|
||||
|
||||
|
||||
def format_docs(docs: list[Document]):
|
||||
"""
|
||||
Format documents for prompt
|
||||
"""
|
||||
"""Format documents for prompt."""
|
||||
return "\n\n".join(doc.page_content for doc in docs)
|
||||
|
||||
|
||||
def create_qa_chain(retriever: Any, llm: ChatOpenAI, prompt: PromptTemplate):
|
||||
"""
|
||||
Set up question answering chain
|
||||
"""
|
||||
"""Set up question answering chain."""
|
||||
return (
|
||||
{
|
||||
"context": retriever | format_docs,
|
||||
@@ -134,9 +122,7 @@ def create_qa_chain(retriever: Any, llm: ChatOpenAI, prompt: PromptTemplate):
|
||||
|
||||
|
||||
def get_parser() -> argparse.ArgumentParser:
|
||||
"""
|
||||
Parse command line arguments
|
||||
"""
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser(description="RAG with vLLM and langchain")
|
||||
|
||||
# Add command line arguments
|
||||
@@ -192,10 +178,7 @@ def get_parser() -> argparse.ArgumentParser:
|
||||
|
||||
|
||||
def init_config(args: Namespace):
|
||||
"""
|
||||
Initialize configuration settings from command line arguments
|
||||
"""
|
||||
|
||||
"""Initialize configuration settings from command line arguments."""
|
||||
return {
|
||||
"vllm_api_key": args.vllm_api_key,
|
||||
"vllm_embedding_endpoint": args.vllm_embedding_endpoint,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
RAG (Retrieval Augmented Generation) Implementation with LlamaIndex
|
||||
"""RAG (Retrieval Augmented Generation) Implementation with LlamaIndex
|
||||
================================================================
|
||||
|
||||
This script demonstrates a RAG system using:
|
||||
@@ -35,6 +34,7 @@ Notes:
|
||||
- Ensure both vLLM services are running before executing
|
||||
- Default ports: 8000 (embedding), 8001 (chat)
|
||||
- First run may take time to download models
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -50,7 +50,7 @@ from llama_index.vector_stores.milvus import MilvusVectorStore
|
||||
|
||||
|
||||
def init_config(args: Namespace):
|
||||
"""Initialize configuration with command line arguments"""
|
||||
"""Initialize configuration with command line arguments."""
|
||||
return {
|
||||
"url": args.url,
|
||||
"embedding_model": args.embedding_model,
|
||||
@@ -66,12 +66,12 @@ def init_config(args: Namespace):
|
||||
|
||||
|
||||
def load_documents(url: str) -> list:
|
||||
"""Load and process web documents"""
|
||||
"""Load and process web documents."""
|
||||
return SimpleWebPageReader(html_to_text=True).load_data([url])
|
||||
|
||||
|
||||
def setup_models(config: dict[str, Any]):
|
||||
"""Configure embedding and chat models"""
|
||||
"""Configure embedding and chat models."""
|
||||
Settings.embed_model = OpenAILikeEmbedding(
|
||||
api_base=config["embedding_endpoint"],
|
||||
api_key=config["vllm_api_key"],
|
||||
@@ -96,14 +96,14 @@ def setup_models(config: dict[str, Any]):
|
||||
|
||||
|
||||
def setup_vector_store(db_path: str) -> MilvusVectorStore:
|
||||
"""Initialize vector store"""
|
||||
"""Initialize vector store."""
|
||||
sample_emb = Settings.embed_model.get_text_embedding("test")
|
||||
print(f"Embedding dimension: {len(sample_emb)}")
|
||||
return MilvusVectorStore(uri=db_path, dim=len(sample_emb), overwrite=True)
|
||||
|
||||
|
||||
def create_index(documents: list, vector_store: MilvusVectorStore):
|
||||
"""Create document index"""
|
||||
"""Create document index."""
|
||||
storage_context = StorageContext.from_defaults(vector_store=vector_store)
|
||||
return VectorStoreIndex.from_documents(
|
||||
documents,
|
||||
@@ -112,13 +112,13 @@ def create_index(documents: list, vector_store: MilvusVectorStore):
|
||||
|
||||
|
||||
def query_document(index: VectorStoreIndex, question: str, top_k: int):
|
||||
"""Query document with given question"""
|
||||
"""Query document with given question."""
|
||||
query_engine = index.as_query_engine(similarity_top_k=top_k)
|
||||
return query_engine.query(question)
|
||||
|
||||
|
||||
def get_parser() -> argparse.ArgumentParser:
|
||||
"""Parse command line arguments"""
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser(description="RAG with vLLM and LlamaIndex")
|
||||
|
||||
# Add command line arguments
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Simple example demonstrating streaming offline inference with AsyncLLM (V1 engine).
|
||||
"""Simple example demonstrating streaming offline inference with AsyncLLM (V1 engine).
|
||||
|
||||
This script shows the core functionality of vLLM's AsyncLLM engine for streaming
|
||||
token-by-token output in offline inference scenarios. It demonstrates DELTA mode
|
||||
@@ -20,8 +19,7 @@ from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
|
||||
async def stream_response(engine: AsyncLLM, prompt: str, request_id: str) -> None:
|
||||
"""
|
||||
Stream response from AsyncLLM and display tokens as they arrive.
|
||||
"""Stream response from AsyncLLM and display tokens as they arrive.
|
||||
|
||||
This function demonstrates the core streaming pattern:
|
||||
1. Create SamplingParams with DELTA output kind
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates using the `LLMEngine`
|
||||
"""This file demonstrates using the `LLMEngine`
|
||||
for processing prompts with various sampling parameters.
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
disagg_encoder_proxy.py
|
||||
"""disagg_encoder_proxy.py
|
||||
|
||||
Proxy that routes OpenAI-compatible “/v1/chat/completions” requests to two
|
||||
clusters:
|
||||
@@ -200,8 +199,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
|
||||
|
||||
def extract_mm_items(request_data: dict) -> list[dict]:
|
||||
"""
|
||||
Return *all* image/audio/video items that appear anywhere in `messages`.
|
||||
"""Return *all* image/audio/video items that appear anywhere in `messages`.
|
||||
|
||||
Each returned dict looks like:
|
||||
{ "type": "image_url", "image_url": {...} }
|
||||
@@ -224,8 +222,7 @@ async def fanout_encoder_primer(
|
||||
req_id: str,
|
||||
consumer_zmq: str | None = None,
|
||||
) -> tuple[dict[int, dict], dict[str, Any]]:
|
||||
"""
|
||||
1. Build one request *per MM item* with all text removed.
|
||||
"""1. Build one request *per MM item* with all text removed.
|
||||
2. Send them concurrently to the encode cluster.
|
||||
3. Raise if any of them fails.
|
||||
|
||||
@@ -402,8 +399,7 @@ async def maybe_prefill(
|
||||
p_url: str,
|
||||
req_id: str,
|
||||
) -> dict:
|
||||
"""
|
||||
- Do prefill-only task if p_url exist;
|
||||
"""- Do prefill-only task if p_url exist;
|
||||
- Return a new body carrying kv transfer params (for nixl connector)
|
||||
- Else, skip and return the original request data for decode
|
||||
|
||||
@@ -849,14 +845,14 @@ async def _post_if_available(
|
||||
payload: dict,
|
||||
headers: dict,
|
||||
) -> dict | None:
|
||||
"""
|
||||
POST `payload` to `url`.
|
||||
"""POST `payload` to `url`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
• The decoded JSON body on success (2xx)
|
||||
• None if the endpoint does not exist (404)
|
||||
• Raises for anything else.
|
||||
|
||||
"""
|
||||
try:
|
||||
resp = await session.post(url, json=payload, headers=headers)
|
||||
@@ -877,9 +873,7 @@ async def _post_if_available(
|
||||
|
||||
|
||||
async def _profile_cmd(cmd: str, payload: dict, e_url: str, p_url: str, d_url: str):
|
||||
"""
|
||||
Fire & forget to both clusters, tolerate 404.
|
||||
"""
|
||||
"""Fire & forget to both clusters, tolerate 404."""
|
||||
headers = {"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY', '')}"}
|
||||
|
||||
encode_task = _post_if_available(
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file provides a disaggregated prefilling proxy demo to demonstrate an
|
||||
"""This file provides a disaggregated prefilling proxy demo to demonstrate an
|
||||
example usage of XpYd disaggregated prefilling.
|
||||
We can launch multiple vllm instances (2 for prefill and 2 for decode), and
|
||||
launch this proxy demo through:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer
|
||||
"""Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer.
|
||||
|
||||
This proxy sits between clients and a vLLM Prefill/Decode (P/D) deployment,
|
||||
routing multi-turn chat requests so that each turn reuses KV cache blocks
|
||||
@@ -54,7 +53,7 @@ Benchmarking:
|
||||
it the benchmark sends OpenAI-schema-compliant payloads and every
|
||||
turn lands as a cache MISS in this proxy.
|
||||
|
||||
Example:
|
||||
Example:
|
||||
python benchmarks/multi_turn/benchmark_serving_multi_turn.py \\
|
||||
--model <MODEL> --served-model-name <NAME> \\
|
||||
--url http://<proxy_host>:8000 \\
|
||||
@@ -67,6 +66,7 @@ Benchmarking:
|
||||
|
||||
Dependencies:
|
||||
pip install fastapi uvicorn httpx
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Push-mode disaggregated prefilling proxy demo.
|
||||
"""Push-mode disaggregated prefilling proxy demo.
|
||||
|
||||
Companion to ``disagg_proxy_demo.py`` (pull mode). The client-facing API is
|
||||
the same; the difference is in how P and D coordinate the KV transfer:
|
||||
|
||||
@@ -6,7 +6,7 @@ from vllm.config import KVTransferConfig
|
||||
|
||||
|
||||
def read_prompts():
|
||||
"""Read prompts from output.txt"""
|
||||
"""Read prompts from output.txt."""
|
||||
prompts = []
|
||||
try:
|
||||
with open("output.txt") as f:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use FlexKV with vLLM for prefix caching.
|
||||
"""This example shows how to use FlexKV with vLLM for prefix caching.
|
||||
|
||||
FlexKV is a distributed KV Store and multi-level cache management system for
|
||||
ultra-large-scale LLM inference.
|
||||
|
||||
@@ -7,7 +7,7 @@ from vllm.config import KVTransferConfig
|
||||
|
||||
|
||||
def read_prompts():
|
||||
"""Read prompts from prefill_output.txt"""
|
||||
"""Read prompts from prefill_output.txt."""
|
||||
prompts = []
|
||||
try:
|
||||
with open("prefill_output.txt") as f:
|
||||
|
||||
+1
-2
@@ -119,8 +119,7 @@ class LoadRecoveryExampleConnector(ExampleConnector):
|
||||
def update_state_after_alloc(
|
||||
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If blocks were allocated, add to _requests_need_load,
|
||||
such that we load the KVs in the next forward pass.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates the example usage of CPU offloading
|
||||
"""This file demonstrates the example usage of CPU offloading
|
||||
with LMCache in vLLM v1.
|
||||
|
||||
Note that `lmcache` is needed to run this example.
|
||||
|
||||
@@ -14,9 +14,7 @@ from fastapi.responses import StreamingResponse
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Lifespan context manager to handle startup and shutdown events.
|
||||
"""
|
||||
"""Lifespan context manager to handle startup and shutdown events."""
|
||||
# Startup: Initialize clients
|
||||
prefiller_base_url = (
|
||||
f"http://{global_args.prefiller_host}:{global_args.prefiller_port}/v1"
|
||||
@@ -106,9 +104,7 @@ app.state.decode_client = None
|
||||
async def send_request_to_service(
|
||||
client: httpx.AsyncClient, endpoint: str, req_data: dict
|
||||
):
|
||||
"""
|
||||
Send a request to a service using a persistent client.
|
||||
"""
|
||||
"""Send a request to a service using a persistent client."""
|
||||
req_data = req_data.copy()
|
||||
req_data["max_tokens"] = 1
|
||||
if "max_completion_tokens" in req_data:
|
||||
@@ -128,9 +124,7 @@ async def send_request_to_service(
|
||||
async def stream_service_response(
|
||||
client: httpx.AsyncClient, endpoint: str, req_data: dict
|
||||
):
|
||||
"""
|
||||
Asynchronously stream the response from a service using a persistent client.
|
||||
"""
|
||||
"""Asynchronously stream the response from a service using a persistent client."""
|
||||
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
|
||||
async with client.stream(
|
||||
"POST", endpoint, json=req_data, headers=headers
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates the example usage of remote KV cache sharing
|
||||
"""This file demonstrates the example usage of remote KV cache sharing
|
||||
with LMCache.
|
||||
We will launch 2 vllm instances, and launch an additional LMCache server.
|
||||
KV cache is transferred in the following manner:
|
||||
|
||||
@@ -65,9 +65,7 @@ async def get_prefiller_info(prefill_clients: list, ready: asyncio.Event):
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Lifespan context manager to handle startup and shutdown events.
|
||||
"""
|
||||
"""Lifespan context manager to handle startup and shutdown events."""
|
||||
# Startup: Initialize client pools for prefiller and decoder services
|
||||
app.state.prefill_clients = []
|
||||
app.state.decode_clients = []
|
||||
@@ -180,10 +178,12 @@ def _parse_prefill_urls(prefill_list):
|
||||
"""Parse prefill URLs from --prefill arguments.
|
||||
|
||||
Format: --prefill URL [BOOTSTRAP_PORT]
|
||||
|
||||
Example:
|
||||
--prefill http://prefill1:8080 9000 # With bootstrap port
|
||||
--prefill http://prefill2:8080 none # Explicitly no bootstrap port
|
||||
--prefill http://prefill3:8080 # Defaults to no bootstrap port
|
||||
|
||||
"""
|
||||
if not prefill_list:
|
||||
return []
|
||||
@@ -228,8 +228,7 @@ def _parse_decode_urls(decode_list):
|
||||
|
||||
|
||||
def get_next_client(app, service_type: str):
|
||||
"""
|
||||
Get the next client in round-robin fashion.
|
||||
"""Get the next client in round-robin fashion.
|
||||
|
||||
Args:
|
||||
app: The FastAPI app instance
|
||||
@@ -237,6 +236,7 @@ def get_next_client(app, service_type: str):
|
||||
|
||||
Returns:
|
||||
The next client to use
|
||||
|
||||
"""
|
||||
if service_type == "prefill":
|
||||
return next(app.state.prefill_iterator)
|
||||
@@ -250,9 +250,7 @@ def get_next_client(app, service_type: str):
|
||||
async def send_request_to_service(
|
||||
client_info: dict, dp_rank: int, endpoint: str, req_data: dict, request_id: str
|
||||
):
|
||||
"""
|
||||
Send a request to a service using a client from the pool.
|
||||
"""
|
||||
"""Send a request to a service using a client from the pool."""
|
||||
req_data = req_data.copy()
|
||||
req_data["kv_transfer_params"] = {
|
||||
"do_remote_decode": True,
|
||||
@@ -288,9 +286,7 @@ async def stream_service_response(
|
||||
req_data: dict,
|
||||
request_id: str,
|
||||
):
|
||||
"""
|
||||
Asynchronously stream response from a service using a client from the pool.
|
||||
"""
|
||||
"""Asynchronously stream response from a service using a client from the pool."""
|
||||
headers = {
|
||||
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
|
||||
"X-Request-Id": request_id,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstration script for Automatic Prefix Caching (APC) in vLLM.
|
||||
"""Demonstration script for Automatic Prefix Caching (APC) in vLLM.
|
||||
|
||||
Automatic Prefix Caching (APC) allows the vLLM engine to reuse cached
|
||||
KV (key-value) pairs from previous prompts if a new query shares the same
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstrates how to achieve reproducibility in vLLM.
|
||||
"""Demonstrates how to achieve reproducibility in vLLM.
|
||||
|
||||
Main article: https://docs.vllm.ai/en/latest/usage/reproducibility.html
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This script demonstrates how to extend the context length
|
||||
"""This script demonstrates how to extend the context length
|
||||
of a Qwen model using the YARN method (rope_parameters)
|
||||
and run a simple chat example.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Usage:
|
||||
"""Usage:
|
||||
Single node:
|
||||
python examples/features/data_parallel/data_parallel_offline.py \
|
||||
--model="ibm-research/PowerMoE-3b" \
|
||||
|
||||
@@ -18,7 +18,7 @@ class EventBatch(msgspec.Struct, array_like=True, omit_defaults=True, gc=False):
|
||||
|
||||
|
||||
class KVCacheEvent(msgspec.Struct, omit_defaults=True, gc=False, tag=True):
|
||||
"""Base class for all KV cache-related events"""
|
||||
"""Base class for all KV cache-related events."""
|
||||
|
||||
|
||||
class BlockStored(KVCacheEvent):
|
||||
|
||||
@@ -48,7 +48,7 @@ from vllm.v1.sample.logits_processor.builtin import process_dict_updates
|
||||
|
||||
# Hypothetical custom logits processor
|
||||
class DummyLogitsProcessor(LogitsProcessor):
|
||||
"""Fake logit processor to support unit testing and examples"""
|
||||
"""Fake logit processor to support unit testing and examples."""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
|
||||
@@ -55,10 +55,11 @@ logger = init_logger(__name__)
|
||||
|
||||
class DummyPerReqLogitsProcessor:
|
||||
"""The request-level logits processor masks out all logits except the
|
||||
token id identified by `target_token`"""
|
||||
token id identified by `target_token`
|
||||
"""
|
||||
|
||||
def __init__(self, target_token: int) -> None:
|
||||
"""Specify `target_token`"""
|
||||
"""Specify `target_token`."""
|
||||
self.target_token = target_token
|
||||
|
||||
def __call__(
|
||||
@@ -74,7 +75,8 @@ class DummyPerReqLogitsProcessor:
|
||||
|
||||
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
"""Example of wrapping a fake request-level logit processor to create a
|
||||
batch-level logits processor"""
|
||||
batch-level logits processor
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
@@ -103,6 +105,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
|
||||
Returns:
|
||||
`Callable` request logits processor, or None
|
||||
|
||||
"""
|
||||
target_token: Any | None = params.extra_args and params.extra_args.get(
|
||||
"target_token"
|
||||
|
||||
@@ -56,10 +56,11 @@ logger = init_logger(__name__)
|
||||
|
||||
class DummyPerReqLogitsProcessor:
|
||||
"""The request-level logits processor masks out all logits except the
|
||||
token id identified by `target_token`"""
|
||||
token id identified by `target_token`
|
||||
"""
|
||||
|
||||
def __init__(self, target_token: int) -> None:
|
||||
"""Specify `target_token`"""
|
||||
"""Specify `target_token`."""
|
||||
self.target_token = target_token
|
||||
|
||||
def __call__(
|
||||
@@ -75,7 +76,8 @@ class DummyPerReqLogitsProcessor:
|
||||
|
||||
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
"""Example of overriding the wrapper class `__init__()` in order to utilize
|
||||
info about the device type"""
|
||||
info about the device type
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
@@ -111,6 +113,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
|
||||
Returns:
|
||||
`Callable` request logits processor, or None
|
||||
|
||||
"""
|
||||
if (
|
||||
not self.is_cuda
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use LoRA with different quantization techniques
|
||||
"""This example shows how to use LoRA with different quantization techniques
|
||||
for offline inference.
|
||||
|
||||
Requires HuggingFace credentials for access.
|
||||
@@ -73,7 +72,6 @@ def initialize_engine(
|
||||
model: str, quantization: str, lora_repo: str | None
|
||||
) -> LLMEngine:
|
||||
"""Initialize the LLMEngine."""
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model=model,
|
||||
quantization=quantization,
|
||||
@@ -86,7 +84,6 @@ def initialize_engine(
|
||||
|
||||
def main():
|
||||
"""Main function that sets up and runs the prompt processing."""
|
||||
|
||||
test_configs = [
|
||||
# QLoRA (https://arxiv.org/abs/2305.14314)
|
||||
{
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use the multi-LoRA functionality
|
||||
"""This example shows how to use the multi-LoRA functionality
|
||||
for offline inference.
|
||||
|
||||
Requires HuggingFace credentials for access to Llama2.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test pause/resume with Data Parallel (DP) via HTTP API.
|
||||
"""Test pause/resume with Data Parallel (DP) via HTTP API.
|
||||
|
||||
This example demonstrates coordinated pause/resume across multiple DP ranks.
|
||||
The pause synchronizes across all DP engines via all-reduce.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstrates how to generate prompt embeddings using
|
||||
"""Demonstrates how to generate prompt embeddings using
|
||||
Hugging Face Transformers and use them as input to vLLM
|
||||
for both single and batch inference.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates preempt requests when using the `LLMEngine`
|
||||
"""This file demonstrates preempt requests when using the `LLMEngine`
|
||||
for processing prompts with various sampling parameters.
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Validates the loading of a model saved with the sharded_state format.
|
||||
"""Validates the loading of a model saved with the sharded_state format.
|
||||
This script demonstrates how to load a model that was previously saved
|
||||
using save_sharded_state_offline.py and validates it by running inference.
|
||||
Example usage:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Saves each worker's model state dict directly to a checkpoint, which enables a
|
||||
"""Saves each worker's model state dict directly to a checkpoint, which enables a
|
||||
fast load path for large tensor-parallel models where each worker only needs to
|
||||
read its own shard rather than the entire checkpoint.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates the usage of text generation with an LLM model,
|
||||
"""This file demonstrates the usage of text generation with an LLM model,
|
||||
comparing the performance with and without speculative decoding.
|
||||
|
||||
Note that this example is out of date and not supported in vLLM v1.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates the example usage of structured outputs
|
||||
"""This file demonstrates the example usage of structured outputs
|
||||
in vLLM. It shows how to apply different constraints such as choice,
|
||||
regex, json schema, and grammar to produce structured and formatted
|
||||
results based on specific prompts.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
experimental support for data-parallel inference with torchrun
|
||||
"""experimental support for data-parallel inference with torchrun
|
||||
Note the data load balancing and distribution is done out of the vllm engine,
|
||||
no internal lb supported in external_launcher mode.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
experimental support for tensor-parallel inference with torchrun,
|
||||
"""experimental support for tensor-parallel inference with torchrun,
|
||||
see https://github.com/vllm-project/vllm/issues/11400 for
|
||||
the motivation and use case for this example.
|
||||
run the script with `torchrun --nproc-per-node=4 torchrun_example_offline.py`,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference
|
||||
"""This example shows how to use vLLM for running offline inference
|
||||
with the correct prompt format on audio language models.
|
||||
|
||||
For most models, the prompt format should follow corresponding examples
|
||||
@@ -286,8 +285,7 @@ def run_minicpmo(question: str, audio_count: int) -> ModelRequestData:
|
||||
|
||||
# Phi-4-multimodal-instruct
|
||||
def run_phi4mm(question: str, audio_count: int) -> ModelRequestData:
|
||||
"""
|
||||
Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
|
||||
"""Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
|
||||
show how to process audio inputs.
|
||||
"""
|
||||
model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference with
|
||||
"""This example shows how to use vLLM for running offline inference with
|
||||
the explicit/implicit prompt format on enc-dec LMMs for text generation.
|
||||
"""
|
||||
|
||||
@@ -56,8 +55,7 @@ def run_whisper():
|
||||
|
||||
|
||||
def run_fireredasr2():
|
||||
"""
|
||||
FireRedASR2 – Automatic Speech Recognition model.
|
||||
"""FireRedASR2 – Automatic Speech Recognition model.
|
||||
|
||||
This model uses a Conformer encoder + Qwen2 LLM decoder architecture
|
||||
for speech-to-text transcription. Audio is passed via the implicit
|
||||
|
||||
@@ -43,7 +43,6 @@ headers = {"User-Agent": "vLLM Example Client"}
|
||||
|
||||
def encode_base64_content_from_url(content_url: str) -> str:
|
||||
"""Encode a content retrieved from a remote url to base64 format."""
|
||||
|
||||
with requests.get(content_url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
result = base64.b64encode(response.content).decode("utf-8")
|
||||
@@ -53,7 +52,6 @@ def encode_base64_content_from_url(content_url: str) -> str:
|
||||
|
||||
def encode_base64_content_from_file(file_path: str) -> str:
|
||||
"""Encode a local file content to base64 format."""
|
||||
|
||||
with open(file_path, "rb") as file:
|
||||
file_content = file.read()
|
||||
result = base64.b64encode(file_content).decode("utf-8")
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference
|
||||
"""This example shows how to use vLLM for running offline inference
|
||||
with the correct prompt format on Qwen2.5-Omni (thinker only).
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference
|
||||
"""This example shows how to use vLLM for running offline inference
|
||||
with the correct prompt format on Qwen3-Omni (thinker only).
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference with
|
||||
"""This example shows how to use vLLM for running offline inference with
|
||||
multi-image input on vision language models for text generation,
|
||||
using the chat template defined by the model.
|
||||
"""
|
||||
@@ -913,11 +912,9 @@ def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData:
|
||||
|
||||
|
||||
def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
|
||||
"""
|
||||
Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
|
||||
"""Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
|
||||
show how to process multi images inputs.
|
||||
"""
|
||||
|
||||
model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
|
||||
# Since the vision-lora and speech-lora co-exist with the base model,
|
||||
# we have to manually specify the path of the lora weights.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference with
|
||||
"""This example shows how to use vLLM for running offline inference with
|
||||
the correct prompt format on vision language models for text generation.
|
||||
|
||||
For most models, the prompt format should follow corresponding examples
|
||||
@@ -1660,8 +1659,7 @@ def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
|
||||
|
||||
# Phi-4-multimodal-instruct
|
||||
def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
|
||||
"""
|
||||
Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
|
||||
"""Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
|
||||
show how to process image inputs.
|
||||
"""
|
||||
assert modality == "image"
|
||||
@@ -2283,8 +2281,7 @@ MODELS_SUPPORT_VIT_CUDA_GRAPH = [
|
||||
|
||||
|
||||
def get_multi_modal_input(args):
|
||||
"""
|
||||
return {
|
||||
"""Return {
|
||||
"data": image or video,
|
||||
"question": question,
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Trace-replay with vLLM offline inference.
|
||||
"""Trace-replay with vLLM offline inference.
|
||||
|
||||
Trace-replay lets you supply a known sequence of decode token IDs alongside
|
||||
the prompt. Instead of sampling from the model distribution, the engine
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Example script demonstrating long text embedding with chunked processing in vLLM.
|
||||
"""Example script demonstrating long text embedding with chunked processing in vLLM.
|
||||
|
||||
This example shows how to use vLLM's chunked processing feature to handle text
|
||||
inputs that exceed the model's maximum token length. The feature automatically
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference with
|
||||
"""This example shows how to use vLLM for running offline inference with
|
||||
the correct prompt format on vision language models for multimodal embedding.
|
||||
|
||||
For most models, the prompt format should follow corresponding examples
|
||||
|
||||
@@ -37,8 +37,7 @@ def create_chat_embeddings(
|
||||
add_special_tokens: bool = False,
|
||||
padding: str | NotGiven = NOT_GIVEN,
|
||||
) -> CreateEmbeddingResponse:
|
||||
"""
|
||||
Convenience function for accessing vLLM's Chat Embeddings API,
|
||||
"""Convenience function for accessing vLLM's Chat Embeddings API,
|
||||
which is an extension of OpenAI's existing Embeddings API.
|
||||
"""
|
||||
return client.post(
|
||||
@@ -56,13 +55,11 @@ def create_chat_embeddings(
|
||||
|
||||
|
||||
def run_clip(client: OpenAI, model: str):
|
||||
"""
|
||||
Start the server using:
|
||||
"""Start the server using:
|
||||
|
||||
vllm serve openai/clip-vit-base-patch32 \
|
||||
--runner pooling
|
||||
"""
|
||||
|
||||
response = create_chat_embeddings(
|
||||
client,
|
||||
messages=[
|
||||
@@ -97,8 +94,7 @@ def run_clip(client: OpenAI, model: str):
|
||||
|
||||
|
||||
def run_dse_qwen2_vl(client: OpenAI, model: str):
|
||||
"""
|
||||
Start the server using:
|
||||
"""Start the server using:
|
||||
|
||||
vllm serve MrLight/dse-qwen2-2b-mrl-v1 \
|
||||
--runner pooling \
|
||||
@@ -159,14 +155,12 @@ def run_dse_qwen2_vl(client: OpenAI, model: str):
|
||||
|
||||
|
||||
def run_qwen3_vl(client: OpenAI, model: str):
|
||||
"""
|
||||
Start the server using:
|
||||
"""Start the server using:
|
||||
|
||||
vllm serve Qwen/Qwen3-VL-Embedding-2B \
|
||||
--runner pooling \
|
||||
--max-model-len 8192
|
||||
"""
|
||||
|
||||
default_instruction = "Represent the user's input."
|
||||
|
||||
print("Text embedding output:")
|
||||
@@ -266,14 +260,12 @@ def run_qwen3_vl(client: OpenAI, model: str):
|
||||
|
||||
|
||||
def run_siglip(client: OpenAI, model: str):
|
||||
"""
|
||||
Start the server using:
|
||||
"""Start the server using:
|
||||
|
||||
vllm serve google/siglip-base-patch16-224 \
|
||||
--runner pooling \
|
||||
--chat-template template_basic.jinja
|
||||
"""
|
||||
|
||||
response = create_chat_embeddings(
|
||||
client,
|
||||
messages=[
|
||||
@@ -311,8 +303,7 @@ def run_siglip(client: OpenAI, model: str):
|
||||
|
||||
|
||||
def run_vlm2vec(client: OpenAI, model: str):
|
||||
"""
|
||||
Start the server using:
|
||||
"""Start the server using:
|
||||
|
||||
vllm serve TIGER-Lab/VLM2Vec-Full \
|
||||
--runner pooling \
|
||||
@@ -320,7 +311,6 @@ def run_vlm2vec(client: OpenAI, model: str):
|
||||
--max-model-len 4096 \
|
||||
--chat-template examples/pooling/embed/template/vlm2vec_phi3v.jinja
|
||||
"""
|
||||
|
||||
response = create_chat_embeddings(
|
||||
client,
|
||||
messages=[
|
||||
|
||||
@@ -85,8 +85,7 @@ def generate_datamodule():
|
||||
|
||||
|
||||
def process_channel_group(orig_img, channels):
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
orig_img: torch.Tensor representing original image (reference)
|
||||
with shape = (bands, H, W).
|
||||
channels: list of indices representing RGB channels.
|
||||
@@ -94,8 +93,8 @@ def process_channel_group(orig_img, channels):
|
||||
Returns:
|
||||
torch.Tensor with shape (num_channels, height, width)
|
||||
for original image
|
||||
"""
|
||||
|
||||
"""
|
||||
orig_img = orig_img[channels, ...]
|
||||
valid_mask = torch.ones_like(orig_img, dtype=torch.bool)
|
||||
valid_mask[orig_img == NO_DATA_FLOAT] = False
|
||||
@@ -121,8 +120,8 @@ def read_geotiff(file_path: str):
|
||||
Returns:
|
||||
np.ndarray with shape (bands, height, width)
|
||||
meta info dict
|
||||
"""
|
||||
|
||||
"""
|
||||
with rasterio.open(file_path) as src:
|
||||
img = src.read()
|
||||
meta = src.meta
|
||||
@@ -142,8 +141,8 @@ def save_geotiff(image, output_path: str, meta: dict):
|
||||
image: np.ndarray with shape (bands, height, width)
|
||||
output_path: path where to save the image
|
||||
meta: dict with meta info.
|
||||
"""
|
||||
|
||||
"""
|
||||
with rasterio.open(output_path, "w", **meta) as dest:
|
||||
for i in range(image.shape[0]):
|
||||
dest.write(image[i, :, :], i + 1)
|
||||
@@ -172,12 +171,14 @@ def load_example(
|
||||
images in *file_paths*.
|
||||
std: list containing std values for each band in the
|
||||
images in *file_paths*.
|
||||
indices: optional band indices to select from each image. All bands
|
||||
are kept when None.
|
||||
|
||||
Returns:
|
||||
np.array containing created example
|
||||
list of meta info for each image in *file_paths*
|
||||
"""
|
||||
|
||||
"""
|
||||
imgs = []
|
||||
metas = []
|
||||
temporal_coords = []
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Example offline usage of sequence reward models.
|
||||
"""Example offline usage of sequence reward models.
|
||||
|
||||
The key distinction between sequence classification and token classification
|
||||
lies in their output granularity: sequence classification produces a single
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example online usage of sequence reward models.
|
||||
"""Example online usage of sequence reward models.
|
||||
|
||||
Run `vllm serve <model> --runner pooling`
|
||||
to start up the server in vLLM. e.g.
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Example offline usage of token reward models.
|
||||
"""Example offline usage of token reward models.
|
||||
|
||||
The key distinction between sequence classification and token classification
|
||||
lies in their output granularity: sequence classification produces a single
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example online usage of token reward models.
|
||||
"""Example online usage of token reward models.
|
||||
|
||||
Run `vllm serve <model> --runner pooling`
|
||||
to start up the server in vLLM. e.g.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example of using the OpenAI entrypoint's rerank API which is compatible with
|
||||
"""Example of using the OpenAI entrypoint's rerank API which is compatible with
|
||||
the Cohere SDK: https://github.com/cohere-ai/cohere-python
|
||||
Note that `pip install cohere` is needed to run this example.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example of using ColBERT late interaction models for reranking and scoring.
|
||||
"""Example of using ColBERT late interaction models for reranking and scoring.
|
||||
|
||||
ColBERT (Contextualized Late Interaction over BERT) uses per-token embeddings
|
||||
and MaxSim scoring for document reranking, providing better accuracy than
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example of using ColModernVBERT late interaction model for reranking.
|
||||
"""Example of using ColModernVBERT late interaction model for reranking.
|
||||
|
||||
ColModernVBERT is a multi-modal ColBERT-style model combining a SigLIP
|
||||
vision encoder with a ModernBERT text encoder. It produces per-token
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example of using ColQwen3.5 late interaction model for reranking.
|
||||
"""Example of using ColQwen3.5 late interaction model for reranking.
|
||||
|
||||
ColQwen3.5 is a multi-modal ColBERT-style model based on Qwen3.5.
|
||||
It produces per-token embeddings and uses MaxSim scoring for retrieval
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
Example of using ColQwen3 late interaction model for reranking and scoring.
|
||||
"""Example of using ColQwen3 late interaction model for reranking and scoring.
|
||||
|
||||
ColQwen3 is a multi-modal ColBERT-style model based on Qwen3-VL.
|
||||
It produces per-token embeddings and uses MaxSim scoring for retrieval
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
"""
|
||||
Script to convert Large Language Models (LLMs) to Sequence Classification models.
|
||||
"""Script to convert Large Language Models (LLMs) to Sequence Classification models.
|
||||
This is particularly useful for converting reranker models that use next-token
|
||||
prediction to a sequence classification format for compatibility with standard
|
||||
classification and rerank pipelines.
|
||||
@@ -35,8 +34,7 @@ import transformers
|
||||
|
||||
|
||||
def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device):
|
||||
"""
|
||||
This method extracts the difference between weights for 'true' and 'false' tokens
|
||||
"""This method extracts the difference between weights for 'true' and 'false' tokens
|
||||
from the language model head to create a single classification weight vector.
|
||||
|
||||
Args:
|
||||
@@ -47,6 +45,7 @@ def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device):
|
||||
device: Target device (cpu/cuda)
|
||||
|
||||
Reference: https://huggingface.co/Qwen/Qwen3-Reranker-0.6B/discussions/3
|
||||
|
||||
"""
|
||||
assert len(tokens) == 2, (
|
||||
"Method requires exactly two tokens for binary classification"
|
||||
@@ -73,8 +72,7 @@ def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device):
|
||||
|
||||
|
||||
def no_post_processing(causal_lm, seq_cls_model, tokenizer, tokens, device):
|
||||
"""
|
||||
Directly use token weights from the language model head for classification.
|
||||
"""Directly use token weights from the language model head for classification.
|
||||
|
||||
This method maps each classification label directly to a corresponding token
|
||||
in the vocabulary without additional transformation.
|
||||
@@ -85,6 +83,7 @@ def no_post_processing(causal_lm, seq_cls_model, tokenizer, tokens, device):
|
||||
tokenizer: Model tokenizer
|
||||
tokens: List of tokens representing class labels
|
||||
device: Target device (cpu/cuda)
|
||||
|
||||
"""
|
||||
# Get the language model head weights (vocabulary_size x hidden_size)
|
||||
lm_head_weights = causal_lm.lm_head.weight
|
||||
@@ -110,8 +109,7 @@ method_map = {
|
||||
def converting(
|
||||
model_name, classifier_from_tokens, path, method, use_sep_token=False, device="cpu"
|
||||
):
|
||||
"""
|
||||
Main conversion function to transform a CausalLM model to SequenceClassification.
|
||||
"""Main conversion function to transform a CausalLM model to SequenceClassification.
|
||||
|
||||
Args:
|
||||
model_name: Name or path of the pretrained model
|
||||
@@ -120,6 +118,7 @@ def converting(
|
||||
method: Conversion method ('from_2_way_softmax' or 'no_post_processing')
|
||||
use_sep_token: Whether to use separating token in the sequence classification model
|
||||
device: Device to load the model on ('cpu' or 'cuda')
|
||||
|
||||
"""
|
||||
assert method in method_map, f"Unknown method: {method}"
|
||||
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
"""
|
||||
What is the difference between the official original version and one
|
||||
"""What is the difference between the official original version and one
|
||||
that has been converted into a sequence classification model?
|
||||
|
||||
Qwen3-Reranker is a language model that doing reranker by using the
|
||||
@@ -34,8 +33,7 @@ model_name = "Qwen/Qwen3-Reranker-0.6B"
|
||||
|
||||
|
||||
def get_llm() -> LLM:
|
||||
"""
|
||||
Initializes and returns the LLM model for Qwen3-Reranker.
|
||||
"""Initializes and returns the LLM model for Qwen3-Reranker.
|
||||
|
||||
Returns:
|
||||
LLM: Configured vLLM instance for reranking tasks.
|
||||
@@ -43,6 +41,7 @@ def get_llm() -> LLM:
|
||||
Note:
|
||||
This function loads the ORIGINAL Qwen3-Reranker model with specific
|
||||
overrides to make it compatible with vLLM's score API.
|
||||
|
||||
"""
|
||||
return LLM(
|
||||
# Specify the original model from HuggingFace
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
What is the difference between the official original version and one
|
||||
"""What is the difference between the official original version and one
|
||||
that has been converted into a sequence classification model?
|
||||
|
||||
Qwen3-Reranker is a language model that doing reranker by using the
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example of using the OpenAI entrypoint's rerank API which is compatible with
|
||||
"""Example of using the OpenAI entrypoint's rerank API which is compatible with
|
||||
Jina and Cohere https://jina.ai/reranker
|
||||
|
||||
run: vllm serve BAAI/bge-reranker-base
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example online usage of Score API.
|
||||
"""Example online usage of Score API.
|
||||
|
||||
Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
|
||||
"""
|
||||
|
||||
@@ -68,13 +68,13 @@ def get_chat_template(model: str) -> str:
|
||||
def get_hf_overrides(model: str) -> dict[str, Any]:
|
||||
"""Convert Large Language Models (LLMs) to Sequence Classification models.
|
||||
|
||||
note:
|
||||
Note:
|
||||
Some reranking models require special configuration overrides to work
|
||||
correctly with vLLM's score API.
|
||||
Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_offline.py
|
||||
Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/convert_model_to_seq_cls.py
|
||||
"""
|
||||
|
||||
"""
|
||||
model_name_to_hf_overrides_map = {
|
||||
"BAAI/bge-reranker-v2-gemma": {
|
||||
"architectures": ["GemmaForSequenceClassification"],
|
||||
@@ -122,7 +122,6 @@ def get_hf_overrides(model: str) -> dict[str, Any]:
|
||||
|
||||
def main(args: Namespace):
|
||||
"""Main execution function for the reranking example."""
|
||||
|
||||
# Get the overrides for the specified model
|
||||
args.hf_overrides = get_hf_overrides(args.model)
|
||||
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
Example of using the rerank API with template.
|
||||
"""Example of using the rerank API with template.
|
||||
|
||||
This script demonstrates how to interact with a vLLM server running
|
||||
a reranking model via the REST API.
|
||||
Before running this script, start the vLLM server with one of the
|
||||
supported reranking models using the commands below.
|
||||
|
||||
note:
|
||||
Note:
|
||||
Some reranking models require special configuration overrides to work correctly
|
||||
with vLLM's score API.
|
||||
Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_online.py
|
||||
@@ -21,6 +20,7 @@ run:
|
||||
vllm serve mixedbread-ai/mxbai-rerank-base-v2 --hf_overrides '{"architectures": ["Qwen2ForSequenceClassification"],"classifier_from_token": ["0", "1"], "method": "from_2_way_softmax"}' --chat-template examples/pooling/score/template/mxbai_rerank_v2.jinja
|
||||
vllm serve nvidia/llama-nemotron-rerank-1b-v2 --runner pooling --trust-remote-code --chat-template examples/pooling/score/template/nemotron-rerank.jinja
|
||||
vllm serve Qwen/Qwen3-Reranker-0.6B --runner pooling --hf_overrides '{"architectures": ["Qwen3ForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}' --chat-template examples/pooling/score/template/qwen3_reranker.jinja
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
"""
|
||||
Example Python client for multimodal rerank API which is compatible with
|
||||
"""Example Python client for multimodal rerank API which is compatible with
|
||||
Jina and Cohere https://jina.ai/reranker
|
||||
|
||||
Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM for running offline inference with
|
||||
"""This example shows how to use vLLM for running offline inference with
|
||||
vision language reranker models for multimodal scoring tasks.
|
||||
|
||||
Vision language rerankers score the relevance between a text query and
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
"""
|
||||
Example online usage of Score API.
|
||||
"""Example online usage of Score API.
|
||||
|
||||
Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
|
||||
e.g.
|
||||
|
||||
@@ -3,8 +3,7 @@
|
||||
# Adapted from Qwen3-ForcedAligner inference:
|
||||
# https://github.com/QwenLM/Qwen3-ASR
|
||||
|
||||
"""
|
||||
Offline forced alignment example using Qwen3-ForcedAligner-0.6B.
|
||||
"""Offline forced alignment example using Qwen3-ForcedAligner-0.6B.
|
||||
|
||||
Forced alignment takes audio and reference text as input and produces
|
||||
word-level timestamps. The model predicts a time bin at each <timestamp>
|
||||
|
||||
@@ -3,8 +3,7 @@
|
||||
# Adapted from Qwen3-ForcedAligner inference:
|
||||
# https://github.com/QwenLM/Qwen3-ASR
|
||||
|
||||
"""
|
||||
Online forced alignment example using Qwen3-ForcedAligner-0.6B-hf.
|
||||
"""Online forced alignment example using Qwen3-ForcedAligner-0.6B-hf.
|
||||
|
||||
Forced alignment takes audio and reference text as input and produces
|
||||
word-level timestamps. The model predicts a time bin at each <timestamp>
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from https://huggingface.co/boltuix/NeuroBERT-NER
|
||||
|
||||
"""
|
||||
Example online usage of Pooling API for Named Entity Recognition (NER).
|
||||
"""Example online usage of Pooling API for Named Entity Recognition (NER).
|
||||
|
||||
Run `vllm serve <model> --runner pooling`
|
||||
to start up the server in vLLM. e.g.
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
"""
|
||||
Example online usage of Pooling API for ColQwen3 multi-vector retrieval.
|
||||
"""Example online usage of Pooling API for ColQwen3 multi-vector retrieval.
|
||||
|
||||
ColQwen3 is a multi-modal late interaction model based on Qwen3-VL that
|
||||
produces per-token embeddings (320-dim, L2-normalized) for both text and
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
"""
|
||||
Example online usage of the Jina Reranker v3 score and rerank APIs with a task
|
||||
"""Example online usage of the Jina Reranker v3 score and rerank APIs with a task
|
||||
instruction.
|
||||
|
||||
Run `vllm serve jinaai/jina-reranker-v3 --runner pooling` to start up the
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Example online usage of Pooling API for multi vector retrieval.
|
||||
"""Example online usage of Pooling API for multi vector retrieval.
|
||||
|
||||
Run `vllm serve <model> --runner pooling`
|
||||
to start up the server in vLLM. e.g.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use Ray Data for data parallel batch inference.
|
||||
"""This example shows how to use Ray Data for data parallel batch inference.
|
||||
|
||||
Ray Data is a data processing framework that can process very large datasets
|
||||
with first-class support for vLLM.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Deploy DeepSeek R1 or V3 with Ray Serve LLM.
|
||||
"""Deploy DeepSeek R1 or V3 with Ray Serve LLM.
|
||||
|
||||
Ray Serve LLM is a scalable and production-grade model serving library built
|
||||
on the Ray distributed computing framework and first-class support for the vLLM engine.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
An example demonstrates how to use tool calling with reasoning models
|
||||
"""An example demonstrates how to use tool calling with reasoning models
|
||||
like QwQ-32B. The reasoning will not be parsed by the tool
|
||||
calling process; only the final output will be parsed.
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
An example shows how to generate chat completions from reasoning models
|
||||
"""An example shows how to generate chat completions from reasoning models
|
||||
like DeepSeekR1.
|
||||
|
||||
To run this example, you need to start the vLLM server
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
An example shows how to generate chat completions from reasoning models
|
||||
"""An example shows how to generate chat completions from reasoning models
|
||||
like DeepSeekR1.
|
||||
|
||||
To run this example, you need to start the vLLM server with the reasoning
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server.
|
||||
"""Set up this example by starting a vLLM OpenAI-compatible server.
|
||||
Reasoning models can be used through the Responses API as seen here
|
||||
https://platform.openai.com/docs/api-reference/responses
|
||||
For example:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstrates async reinforcement learning using vLLM and Ray,
|
||||
"""Demonstrates async reinforcement learning using vLLM and Ray,
|
||||
with native weight syncing APIs and batch-invariant generation.
|
||||
|
||||
The script separates training and inference workloads onto distinct GPUs
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the
|
||||
"""RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the
|
||||
data plane.
|
||||
|
||||
* OpenAI-compatible API for inference requests
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the
|
||||
"""RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the
|
||||
data plane.
|
||||
|
||||
* OpenAI-compatible API for inference requests
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC**
|
||||
"""RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC**
|
||||
weight transfer and **packed** tensors.
|
||||
|
||||
Multi-rank version of `rlhf_http_ipc.py`: the trainer is 4 FSDP2 Ray actors
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs).
|
||||
"""RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs).
|
||||
|
||||
8-GPU layout:
|
||||
Training — 4 GPUs, PyTorch FSDP2 (fully_shard), as Ray actors
|
||||
@@ -74,8 +73,7 @@ BASE_URL = f"http://localhost:{SERVER_PORT}"
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
class FSDPTrainWorker:
|
||||
"""
|
||||
One FSDP2 training worker per GPU. Four of these form the FSDP group.
|
||||
"""One FSDP2 training worker per GPU. Four of these form the FSDP group.
|
||||
Rank 0 additionally drives weight transfer to the vLLM server.
|
||||
"""
|
||||
|
||||
|
||||
@@ -101,7 +101,8 @@ PROMPTS = ["The capital of France is", "The future of AI is"]
|
||||
class FSDPTrainWorker:
|
||||
"""One FSDP2 rank per GPU. Identical to the 8-GPU example's worker: the
|
||||
sharded-RDT engine owns the NIXL serve surface, so this stays a plain Ray
|
||||
actor with no producer mixin."""
|
||||
actor with no producer mixin.
|
||||
"""
|
||||
|
||||
def __init__(self, model_name, rank, world_size, master_addr, master_port):
|
||||
self.rank = rank
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
End-to-end example for routed experts capture with hybrid models.
|
||||
"""End-to-end example for routed experts capture with hybrid models.
|
||||
|
||||
Validates that:
|
||||
1. routed_experts is returned in CompletionOutput for MoE models.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This script demonstrates how to use the vLLM API server to perform audio
|
||||
"""This script demonstrates how to use the vLLM API server to perform audio
|
||||
transcription with the `openai/whisper-large-v3` model.
|
||||
|
||||
Before running this script, you must start the vLLM server with the following command:
|
||||
@@ -35,8 +34,7 @@ def sync_openai(
|
||||
hotwords: str = None,
|
||||
prompt: str | None = None,
|
||||
):
|
||||
"""
|
||||
Perform synchronous transcription using OpenAI-compatible API.
|
||||
"""Perform synchronous transcription using OpenAI-compatible API.
|
||||
|
||||
The optional ``prompt`` is the OpenAI-API ``prompt`` field (style /
|
||||
vocabulary hint). It is wired through model-by-model: Whisper uses it
|
||||
@@ -69,9 +67,7 @@ async def stream_openai_response(
|
||||
hotwords: str = None,
|
||||
prompt: str | None = None,
|
||||
):
|
||||
"""
|
||||
Perform asynchronous transcription using OpenAI-compatible API.
|
||||
"""
|
||||
"""Perform asynchronous transcription using OpenAI-compatible API."""
|
||||
print("\ntranscription result [stream]:", end=" ")
|
||||
with open(audio_path, "rb") as f:
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
@@ -98,9 +94,7 @@ async def stream_openai_response(
|
||||
|
||||
|
||||
def stream_api_response(audio_path: str, model: str, openai_api_base: str):
|
||||
"""
|
||||
Perform streaming transcription using raw HTTP requests to the vLLM API server.
|
||||
"""
|
||||
"""Stream a transcription using raw HTTP requests to the vLLM API server."""
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This script demonstrates how to use the vLLM Realtime WebSocket API to perform
|
||||
"""This script demonstrates how to use the vLLM Realtime WebSocket API to perform
|
||||
audio transcription by uploading an audio file.
|
||||
|
||||
Before running this script, you must start the vLLM server with a realtime-capable
|
||||
@@ -34,9 +33,7 @@ from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
|
||||
def audio_to_pcm16_base64(audio_path: str) -> str:
|
||||
"""
|
||||
Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
|
||||
"""
|
||||
"""Load an audio file and convert it to base64-encoded PCM16 @ 16kHz."""
|
||||
# Load audio and resample to 16kHz mono
|
||||
audio, _ = load_audio(audio_path, sr=16000, mono=True)
|
||||
# Convert to PCM16
|
||||
@@ -46,9 +43,7 @@ def audio_to_pcm16_base64(audio_path: str) -> str:
|
||||
|
||||
|
||||
async def realtime_transcribe(audio_path: str, host: str, port: int, model: str):
|
||||
"""
|
||||
Connect to the Realtime API and transcribe an audio file.
|
||||
"""
|
||||
"""Connect to the Realtime API and transcribe an audio file."""
|
||||
uri = f"ws://{host}:{port}/v1/realtime"
|
||||
|
||||
async with websockets.connect(uri) as ws:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API.
|
||||
"""Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API.
|
||||
|
||||
Start the vLLM server first:
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled. For example:
|
||||
|
||||
IMPORTANT: for mistral, you must use one of the provided mistral tool call
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
To run this example, you can start the vLLM server
|
||||
"""To run this example, you can start the vLLM server
|
||||
without any specific flags:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled for xLAM-2 models:
|
||||
|
||||
vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
|
||||
@@ -105,8 +104,7 @@ tool_functions = {
|
||||
|
||||
|
||||
def process_response(response, tool_functions, original_query):
|
||||
"""Process a non-streaming response with possible tool calls"""
|
||||
|
||||
"""Process a non-streaming response with possible tool calls."""
|
||||
print("\n--- Response Output ---")
|
||||
|
||||
# Check if the response has content
|
||||
@@ -195,7 +193,7 @@ def process_response(response, tool_functions, original_query):
|
||||
|
||||
|
||||
def run_test_case(query, test_name):
|
||||
"""Run a single test case with the given query"""
|
||||
"""Run a single test case with the given query."""
|
||||
print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}")
|
||||
print(f"Query: '{query}'")
|
||||
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled for xLAM-2 models:
|
||||
|
||||
vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam
|
||||
@@ -107,7 +106,7 @@ tool_functions = {
|
||||
|
||||
|
||||
def process_stream(response, tool_functions, original_query):
|
||||
"""Process a streaming response with possible tool calls"""
|
||||
"""Process a streaming response with possible tool calls."""
|
||||
# Track multiple tool calls
|
||||
tool_calls = {} # Dictionary to store tool calls by ID
|
||||
|
||||
@@ -223,7 +222,7 @@ def process_stream(response, tool_functions, original_query):
|
||||
|
||||
|
||||
def run_test_case(query, test_name):
|
||||
"""Run a single test case with the given query"""
|
||||
"""Run a single test case with the given query."""
|
||||
print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}")
|
||||
print(f"Query: '{query}'")
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Example demonstrating MCP (Model Context Protocol) tools with the Responses API.
|
||||
"""Example demonstrating MCP (Model Context Protocol) tools with the Responses API.
|
||||
|
||||
This example shows how to use MCP tools with different allowed_tools configurations:
|
||||
1. No filter (allows all tools from the MCP server)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
"""Set up this example by starting a vLLM OpenAI-compatible server with tool call
|
||||
options enabled.
|
||||
Reasoning models can be used through the Responses API as seen here
|
||||
https://platform.openai.com/docs/api-reference/responses
|
||||
@@ -17,8 +16,7 @@ from openai import OpenAI
|
||||
|
||||
|
||||
def get_weather(latitude: float, longitude: float) -> str:
|
||||
"""
|
||||
Mock function to simulate getting weather data.
|
||||
"""Mock function to simulate getting weather data.
|
||||
In a real application, this would call an external weather API.
|
||||
"""
|
||||
return f"Current temperature at ({latitude}, {longitude}) is 20°C."
|
||||
|
||||
Reference in New Issue
Block a user