Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,8 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
disagg_encoder_proxy.py
|
||||
"""disagg_encoder_proxy.py
|
||||
|
||||
Proxy that routes OpenAI-compatible “/v1/chat/completions” requests to two
|
||||
clusters:
|
||||
@@ -200,8 +199,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
|
||||
|
||||
def extract_mm_items(request_data: dict) -> list[dict]:
|
||||
"""
|
||||
Return *all* image/audio/video items that appear anywhere in `messages`.
|
||||
"""Return *all* image/audio/video items that appear anywhere in `messages`.
|
||||
|
||||
Each returned dict looks like:
|
||||
{ "type": "image_url", "image_url": {...} }
|
||||
@@ -224,8 +222,7 @@ async def fanout_encoder_primer(
|
||||
req_id: str,
|
||||
consumer_zmq: str | None = None,
|
||||
) -> tuple[dict[int, dict], dict[str, Any]]:
|
||||
"""
|
||||
1. Build one request *per MM item* with all text removed.
|
||||
"""1. Build one request *per MM item* with all text removed.
|
||||
2. Send them concurrently to the encode cluster.
|
||||
3. Raise if any of them fails.
|
||||
|
||||
@@ -402,8 +399,7 @@ async def maybe_prefill(
|
||||
p_url: str,
|
||||
req_id: str,
|
||||
) -> dict:
|
||||
"""
|
||||
- Do prefill-only task if p_url exist;
|
||||
"""- Do prefill-only task if p_url exist;
|
||||
- Return a new body carrying kv transfer params (for nixl connector)
|
||||
- Else, skip and return the original request data for decode
|
||||
|
||||
@@ -849,14 +845,14 @@ async def _post_if_available(
|
||||
payload: dict,
|
||||
headers: dict,
|
||||
) -> dict | None:
|
||||
"""
|
||||
POST `payload` to `url`.
|
||||
"""POST `payload` to `url`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
• The decoded JSON body on success (2xx)
|
||||
• None if the endpoint does not exist (404)
|
||||
• Raises for anything else.
|
||||
|
||||
"""
|
||||
try:
|
||||
resp = await session.post(url, json=payload, headers=headers)
|
||||
@@ -877,9 +873,7 @@ async def _post_if_available(
|
||||
|
||||
|
||||
async def _profile_cmd(cmd: str, payload: dict, e_url: str, p_url: str, d_url: str):
|
||||
"""
|
||||
Fire & forget to both clusters, tolerate 404.
|
||||
"""
|
||||
"""Fire & forget to both clusters, tolerate 404."""
|
||||
headers = {"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY', '')}"}
|
||||
|
||||
encode_task = _post_if_available(
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file provides a disaggregated prefilling proxy demo to demonstrate an
|
||||
"""This file provides a disaggregated prefilling proxy demo to demonstrate an
|
||||
example usage of XpYd disaggregated prefilling.
|
||||
We can launch multiple vllm instances (2 for prefill and 2 for decode), and
|
||||
launch this proxy demo through:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer
|
||||
"""Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer.
|
||||
|
||||
This proxy sits between clients and a vLLM Prefill/Decode (P/D) deployment,
|
||||
routing multi-turn chat requests so that each turn reuses KV cache blocks
|
||||
@@ -54,7 +53,7 @@ Benchmarking:
|
||||
it the benchmark sends OpenAI-schema-compliant payloads and every
|
||||
turn lands as a cache MISS in this proxy.
|
||||
|
||||
Example:
|
||||
Example:
|
||||
python benchmarks/multi_turn/benchmark_serving_multi_turn.py \\
|
||||
--model <MODEL> --served-model-name <NAME> \\
|
||||
--url http://<proxy_host>:8000 \\
|
||||
@@ -67,6 +66,7 @@ Benchmarking:
|
||||
|
||||
Dependencies:
|
||||
pip install fastapi uvicorn httpx
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Push-mode disaggregated prefilling proxy demo.
|
||||
"""Push-mode disaggregated prefilling proxy demo.
|
||||
|
||||
Companion to ``disagg_proxy_demo.py`` (pull mode). The client-facing API is
|
||||
the same; the difference is in how P and D coordinate the KV transfer:
|
||||
|
||||
@@ -6,7 +6,7 @@ from vllm.config import KVTransferConfig
|
||||
|
||||
|
||||
def read_prompts():
|
||||
"""Read prompts from output.txt"""
|
||||
"""Read prompts from output.txt."""
|
||||
prompts = []
|
||||
try:
|
||||
with open("output.txt") as f:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use FlexKV with vLLM for prefix caching.
|
||||
"""This example shows how to use FlexKV with vLLM for prefix caching.
|
||||
|
||||
FlexKV is a distributed KV Store and multi-level cache management system for
|
||||
ultra-large-scale LLM inference.
|
||||
|
||||
@@ -7,7 +7,7 @@ from vllm.config import KVTransferConfig
|
||||
|
||||
|
||||
def read_prompts():
|
||||
"""Read prompts from prefill_output.txt"""
|
||||
"""Read prompts from prefill_output.txt."""
|
||||
prompts = []
|
||||
try:
|
||||
with open("prefill_output.txt") as f:
|
||||
|
||||
+1
-2
@@ -119,8 +119,7 @@ class LoadRecoveryExampleConnector(ExampleConnector):
|
||||
def update_state_after_alloc(
|
||||
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If blocks were allocated, add to _requests_need_load,
|
||||
such that we load the KVs in the next forward pass.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates the example usage of CPU offloading
|
||||
"""This file demonstrates the example usage of CPU offloading
|
||||
with LMCache in vLLM v1.
|
||||
|
||||
Note that `lmcache` is needed to run this example.
|
||||
|
||||
@@ -14,9 +14,7 @@ from fastapi.responses import StreamingResponse
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Lifespan context manager to handle startup and shutdown events.
|
||||
"""
|
||||
"""Lifespan context manager to handle startup and shutdown events."""
|
||||
# Startup: Initialize clients
|
||||
prefiller_base_url = (
|
||||
f"http://{global_args.prefiller_host}:{global_args.prefiller_port}/v1"
|
||||
@@ -106,9 +104,7 @@ app.state.decode_client = None
|
||||
async def send_request_to_service(
|
||||
client: httpx.AsyncClient, endpoint: str, req_data: dict
|
||||
):
|
||||
"""
|
||||
Send a request to a service using a persistent client.
|
||||
"""
|
||||
"""Send a request to a service using a persistent client."""
|
||||
req_data = req_data.copy()
|
||||
req_data["max_tokens"] = 1
|
||||
if "max_completion_tokens" in req_data:
|
||||
@@ -128,9 +124,7 @@ async def send_request_to_service(
|
||||
async def stream_service_response(
|
||||
client: httpx.AsyncClient, endpoint: str, req_data: dict
|
||||
):
|
||||
"""
|
||||
Asynchronously stream the response from a service using a persistent client.
|
||||
"""
|
||||
"""Asynchronously stream the response from a service using a persistent client."""
|
||||
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
|
||||
async with client.stream(
|
||||
"POST", endpoint, json=req_data, headers=headers
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file demonstrates the example usage of remote KV cache sharing
|
||||
"""This file demonstrates the example usage of remote KV cache sharing
|
||||
with LMCache.
|
||||
We will launch 2 vllm instances, and launch an additional LMCache server.
|
||||
KV cache is transferred in the following manner:
|
||||
|
||||
@@ -65,9 +65,7 @@ async def get_prefiller_info(prefill_clients: list, ready: asyncio.Event):
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Lifespan context manager to handle startup and shutdown events.
|
||||
"""
|
||||
"""Lifespan context manager to handle startup and shutdown events."""
|
||||
# Startup: Initialize client pools for prefiller and decoder services
|
||||
app.state.prefill_clients = []
|
||||
app.state.decode_clients = []
|
||||
@@ -180,10 +178,12 @@ def _parse_prefill_urls(prefill_list):
|
||||
"""Parse prefill URLs from --prefill arguments.
|
||||
|
||||
Format: --prefill URL [BOOTSTRAP_PORT]
|
||||
|
||||
Example:
|
||||
--prefill http://prefill1:8080 9000 # With bootstrap port
|
||||
--prefill http://prefill2:8080 none # Explicitly no bootstrap port
|
||||
--prefill http://prefill3:8080 # Defaults to no bootstrap port
|
||||
|
||||
"""
|
||||
if not prefill_list:
|
||||
return []
|
||||
@@ -228,8 +228,7 @@ def _parse_decode_urls(decode_list):
|
||||
|
||||
|
||||
def get_next_client(app, service_type: str):
|
||||
"""
|
||||
Get the next client in round-robin fashion.
|
||||
"""Get the next client in round-robin fashion.
|
||||
|
||||
Args:
|
||||
app: The FastAPI app instance
|
||||
@@ -237,6 +236,7 @@ def get_next_client(app, service_type: str):
|
||||
|
||||
Returns:
|
||||
The next client to use
|
||||
|
||||
"""
|
||||
if service_type == "prefill":
|
||||
return next(app.state.prefill_iterator)
|
||||
@@ -250,9 +250,7 @@ def get_next_client(app, service_type: str):
|
||||
async def send_request_to_service(
|
||||
client_info: dict, dp_rank: int, endpoint: str, req_data: dict, request_id: str
|
||||
):
|
||||
"""
|
||||
Send a request to a service using a client from the pool.
|
||||
"""
|
||||
"""Send a request to a service using a client from the pool."""
|
||||
req_data = req_data.copy()
|
||||
req_data["kv_transfer_params"] = {
|
||||
"do_remote_decode": True,
|
||||
@@ -288,9 +286,7 @@ async def stream_service_response(
|
||||
req_data: dict,
|
||||
request_id: str,
|
||||
):
|
||||
"""
|
||||
Asynchronously stream response from a service using a client from the pool.
|
||||
"""
|
||||
"""Asynchronously stream response from a service using a client from the pool."""
|
||||
headers = {
|
||||
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
|
||||
"X-Request-Id": request_id,
|
||||
|
||||
Reference in New Issue
Block a user