Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+1 -2
View File
@@ -219,8 +219,7 @@ class DFlashProposer(SpecDecodeBaseProposer):
is_graph_capturing: bool = False,
slot_mappings: dict[str, torch.Tensor] | None = None,
) -> None:
"""
Key differences to default dummy_run:
"""Key differences to default dummy_run:
- Only one forward pass due to parallel drafting
- DFlash uses context states as unpadded metadata, so hidden_states will
use the unpadded num_tokens instead of num_input_tokens
+6 -4
View File
@@ -106,6 +106,7 @@ class ExtractHiddenStatesProposer:
The main purpose is to cache hidden states, not to speculate.
Args:
num_speculative_tokens: Number of draft tokens to return per request
sampled_token_ids: Sampled token IDs from the target model
target_hidden_states: List of hidden state tensors from target model
(one per aux hidden state layer)
@@ -117,6 +118,7 @@ class ExtractHiddenStatesProposer:
Tuple of:
- Draft tokens matching sampled tokens, shape [batch_size, 1]
- KV connector output (if KV transfer is active), else None
"""
assert num_speculative_tokens == self.num_speculative_tokens
assert self.model is not None and isinstance(target_hidden_states, list)
@@ -318,14 +320,12 @@ class ExtractHiddenStatesProposer:
gpu_input_batch: InputBatch,
discard_request_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Prepare next token IDs for speculative decoding.
"""Prepare next token IDs for speculative decoding.
Since num_speculative_tokens == 1, sampled_token_ids has shape
(batch_size, 1). For each request we either use the sampled token
(if valid and not discarded) or a backup token from the request state.
"""
# Precompute backup token IDs for discarded requests.
num_reqs = gpu_input_batch.num_reqs
for i in range(num_reqs):
@@ -359,6 +359,7 @@ class ExtractHiddenStatesProposer:
Args:
target_model: The target model (passed for compatibility with
EagleProposer interface, but not used here)
"""
# Get the target model's attention layers before loading draft model
target_attn_layer_names = set(
@@ -395,7 +396,8 @@ class ExtractHiddenStatesProposer:
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
"""Validate all drafting layers belong to the same KV cache group
and record the group index for common_attn_metadata selection."""
and record the group index for common_attn_metadata selection.
"""
assert len(self.attn_layer_names) == 1
layer = self.attn_layer_names[0]
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
+4 -2
View File
@@ -221,7 +221,8 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
"""Draft layers span multiple KV cache groups (sliding + full
attention with different head dimensions), so skip the base
class single-group assertion."""
class single-group assertion.
"""
def initialize_attn_backend(
self,
@@ -229,7 +230,8 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
kernel_block_sizes: list[int] | None = None,
) -> None:
"""Create separate AttentionGroup objects per KV cache spec
so that each head-dim variant gets its own metadata builder."""
so that each head-dim variant gets its own metadata builder.
"""
all_attn_layers = get_layers_from_vllm_config(
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
+12 -21
View File
@@ -785,7 +785,8 @@ class SpecDecodeBaseProposer:
block_size: int,
) -> torch.Tensor:
"""Update positions, slot mappings, and sequence metadata for the
next draft step. Returns the updated positions tensor."""
next draft step. Returns the updated positions tensor.
"""
positions_1d = positions[0] if self.uses_mrope else positions
if self.uses_mrope:
out_pos = self.mrope_positions[0, :batch_size]
@@ -1024,8 +1025,7 @@ class SpecDecodeBaseProposer:
gpu_input_batch: InputBatch,
num_scheduled_tokens: dict[str, int],
) -> torch.Tensor:
"""
This function is used to prepare the inputs for speculative decoding.
"""This function is used to prepare the inputs for speculative decoding.
It calculates the next token ids for each request based on the sampled
token ids from the CPU. If a request has no sampled token ids (e.g.,
during the initial decoding steps), it falls back to using the request
@@ -1057,8 +1057,7 @@ class SpecDecodeBaseProposer:
gpu_input_batch: InputBatch,
discard_request_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
This function is used to prepare the inputs for speculative decoding.
"""This function is used to prepare the inputs for speculative decoding.
It calculates the next token ids and the number of valid sampled tokens
for each request, considering the "discarded" requests whose next token
is not sampled and comes from `request.get_token_id()` instead. This is denoted
@@ -1107,8 +1106,7 @@ class SpecDecodeBaseProposer:
spec_decode_metadata: SpecDecodeMetadata,
valid_sampled_tokens_count: torch.Tensor,
) -> tuple[CommonAttentionMetadata, torch.Tensor, torch.Tensor]:
"""
This function is used to prepare the inputs for speculative decoding
"""This function is used to prepare the inputs for speculative decoding
It updates the common_attn_metadata for speculative decoding,
but does not consider the rejected tokens. Instead, all tokens
are included as inputs to the speculator, with the rejected tokens
@@ -1166,8 +1164,7 @@ class SpecDecodeBaseProposer:
sampled_token_ids: list[list[int]],
num_draft_tokens: list[int],
) -> tuple[CommonAttentionMetadata, torch.Tensor]:
"""
This function is used to prepare the inputs for speculative decoding.
"""This function is used to prepare the inputs for speculative decoding.
It updates to the common_attn_metadata to account for the rejected
tokens (and newly sampled tokens). It also returns the token indices
of the tokens that should be fed to the speculator.
@@ -1309,8 +1306,7 @@ class SpecDecodeBaseProposer:
return base
def _get_model(self) -> nn.Module:
"""
Default method to call get_model(). Can be overridden by subclasses which
"""Default method to call get_model(). Can be overridden by subclasses which
need to customize model loading.
"""
from vllm.compilation.backends import set_model_tag
@@ -1422,8 +1418,7 @@ class SpecDecodeBaseProposer:
self.parallel_drafting_hidden_state_tensor.copy_(flat_mask)
def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None:
"""
Some draft models may not have their own embedding layers, and some may
"""Some draft models may not have their own embedding layers, and some may
have a duplicate copy of the target model's embedding layers. In these cases,
we share the target model's embedding layers with the draft model to save
memory.
@@ -1515,8 +1510,7 @@ class SpecDecodeBaseProposer:
)
def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None:
"""
Some draft models may not have their own LM head, and some may have a
"""Some draft models may not have their own LM head, and some may have a
duplicate copy of the target model's LM head. In these cases, we share
the target model's LM head with the draft model to save memory.
"""
@@ -1680,8 +1674,7 @@ class SpecDecodeBaseProposer:
self.model(**kwargs)
def _get_eagle3_use_aux_hidden_state_from_config(self) -> bool:
"""
Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary
"""Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary
hidden states and directly uses the last layer output just like eagle1.
They might indicate this by setting "use_aux_hidden_state" to False
inside the "eagle_config" dict of their hf_config.
@@ -1696,8 +1689,7 @@ class SpecDecodeBaseProposer:
return use_aux_hidden_state
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
"""
Validate that all drafting layers belong to the same KVCacheGroup.
"""Validate that all drafting layers belong to the same KVCacheGroup.
Need this assumption to ensure all drafting layers can use the
same AttentionMetadata.
May extend to multiple AttentionMetadata in the future.
@@ -1723,8 +1715,7 @@ class SpecDecodeBaseProposer:
kv_cache_config: KVCacheConfig,
kernel_block_sizes: list[int] | None = None,
) -> None:
"""
Initialize AttentionGroups for draft layers using kv_cache_config.
"""Initialize AttentionGroups for draft layers using kv_cache_config.
Called from the model runner's initialize_metadata_builders.
"""
all_attn_layers = get_layers_from_vllm_config(
+1 -3
View File
@@ -16,9 +16,7 @@ logger = init_logger(__name__)
class MedusaProposer:
"""
Medusa proposer class for generating token sequences
"""
"""Medusa proposer class for generating token sequences."""
def __init__(
self,
+4 -2
View File
@@ -72,6 +72,8 @@ class NgramProposer:
"""Batch version of ngram proposer using numba for acceleration.
Args:
num_requests:
Number of requests in the batch.
valid_ngram_requests:
Set of indices of requests that need ngram proposals.
num_tokens_no_spec:
@@ -87,6 +89,7 @@ class NgramProposer:
list[list[int]]:
A list where each element is a list of proposed
token IDs for the corresponding request.
"""
draft_token_ids: list[list[int]] = []
@@ -211,8 +214,7 @@ def _find_longest_matched_ngram_and_propose_tokens(
max_model_len: int,
k: int,
) -> np.ndarray:
"""
Find the longest n-gram which matches the suffix of the given tokens
"""Find the longest n-gram which matches the suffix of the given tokens
whose length is within [min_ngram, max_ngram] (inclusive).
If found, we will extract k right after the matched ngram.
+12 -16
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
GPU-accelerated N-gram proposer using fully async PyTorch tensor operations.
"""GPU-accelerated N-gram proposer using fully async PyTorch tensor operations.
This version uses a fully vectorized approach with unfold and argmax for
finding the first match across all sequences in parallel.
@@ -52,8 +51,7 @@ class NgramGPUKernel(nn.Module):
max_ngram_len: int,
num_draft_tokens: int,
) -> torch.Tensor:
"""
Find suffix n-gram matches and extract following tokens.
"""Find suffix n-gram matches and extract following tokens.
Searches for the earliest prior occurrence of the trailing n-gram,
tries multiple lengths, and picks the longest valid match.
@@ -66,6 +64,7 @@ class NgramGPUKernel(nn.Module):
Returns:
Draft token predictions; -1 means invalid/no match.
"""
batch_size = token_ids.shape[0]
max_seq_len = token_ids.shape[1]
@@ -165,8 +164,7 @@ class NgramGPUKernel(nn.Module):
token_ids_gpu: torch.Tensor,
combined_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Forward pass for N-gram proposal using GPU tensor operations.
"""Forward pass for N-gram proposal using GPU tensor operations.
Args:
num_tokens_no_spec: Number of tokens for each sequence [batch_size]
@@ -177,8 +175,8 @@ class NgramGPUKernel(nn.Module):
draft_tokens: [batch_size, k] on GPU
num_valid_draft_tokens: [batch_size] int32 on GPU, count of
leading valid (non -1) tokens per request.
"""
"""
device = token_ids_gpu.device
# Infer batch size to preserve dynamic shape.
@@ -282,8 +280,7 @@ class NgramProposerGPU:
pattern_len: int,
device: str = "cuda",
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Generate random test data with n-gram repetitions.
"""Generate random test data with n-gram repetitions.
Args:
batch_size: Number of sequences in the batch
@@ -296,6 +293,7 @@ class NgramProposerGPU:
num_tokens: [batch_size] tensor
sampled_flags: [batch_size] bool tensor
valid_mask: [batch_size] bool tensor
"""
token_ids = torch.zeros(
batch_size,
@@ -321,8 +319,7 @@ class NgramProposerGPU:
valid_sampled_token_ids_gpu: torch.Tensor, # [batch_size, num_spec_tokens + 1]
valid_sampled_tokens_count: torch.Tensor, # [batch_size]
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Propose draft tokens using GPU-accelerated n-gram matching.
"""Propose draft tokens using GPU-accelerated n-gram matching.
Scatter sampled tokens into `token_ids_gpu`, compute temporary
updated lengths, then run the kernel.
@@ -338,6 +335,7 @@ class NgramProposerGPU:
draft_tokens: Proposed draft token IDs [batch_size, k]
num_valid_draft_tokens: Count of leading valid draft tokens
per request [batch_size]
"""
assert num_speculative_tokens == self.k
assert token_ids_gpu.device == self.device
@@ -398,8 +396,7 @@ class NgramProposerGPU:
num_tokens_no_spec: torch.Tensor,
discard_request_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Prepare speculative decoding inputs on device:
"""Prepare speculative decoding inputs on device:
compute next token ids and valid counts, honoring discarded requests
and rejected tokens, without CPU-GPU sync.
"""
@@ -485,6 +482,7 @@ def update_scheduler_for_invalid_drafts(
num_valid_draft_tokens_cpu: CPU buffer of valid draft counts.
scheduler_output: Scheduler metadata to update in-place.
req_id_to_index: Request-id to batch-index mapping.
"""
req_data = scheduler_output.scheduled_cached_reqs
num_valid_draft_tokens_event.synchronize()
@@ -652,9 +650,7 @@ def copy_num_valid_draft_tokens(
num_valid_draft_tokens: torch.Tensor | None,
batch_size: int,
) -> None:
"""
Async D2H copy of per-request valid draft counts.
"""
"""Async D2H copy of per-request valid draft counts."""
if num_valid_draft_tokens is None:
return
-1
View File
@@ -155,7 +155,6 @@ class Step3p5MTPProposer(EagleProposer):
def _maybe_share_lm_head(self, target_language_model: torch.nn.Module) -> None:
"""Step3.5 MTP uses the lm_head stored in each MTP layer."""
# The base MTP path shares target lm_head into shared_head.head.
# Step3.5 checkpoints carry per-MTP-layer shared_head weights.
return
+1 -2
View File
@@ -7,8 +7,7 @@ from vllm.v1.worker.gpu_input_batch import InputBatch
class SuffixDecodingProposer:
"""
Speculative decoding proposer for Suffix Decoding (https://arxiv.org/pdf/2411.04975).
"""Speculative decoding proposer for Suffix Decoding (https://arxiv.org/pdf/2411.04975).
This class imports and uses the official implementation from Arctic Inference
(https://github.com/snowflakedb/ArcticInference).
"""
+10 -15
View File
@@ -50,8 +50,7 @@ def eagle_step_slot_mapping_metadata_kernel(
PAD_ID: tl.constexpr,
batch_size,
):
"""
Fused kernel for EAGLE autoregressive step: updates positions, slot mapping,
"""Fused kernel for EAGLE autoregressive step: updates positions, slot mapping,
and sequence lengths in a single kernel to reduce launch overhead.
Launched with input_batch_size threads. Threads with req_idx >= batch_size
@@ -149,8 +148,7 @@ def eagle_step_update_slot_mapping_and_metadata(
out_slot_mapping: torch.Tensor,
input_batch_size: int | None = None,
) -> None:
"""
Fused update of slot mapping and metadata for one EAGLE autoregressive step.
"""Fused update of slot mapping and metadata for one EAGLE autoregressive step.
Updates seq_lens in place. Writes to out_clamped_positions and out_slot_mapping.
When input_batch_size > batch_size, threads beyond batch_size write
@@ -166,6 +164,7 @@ def eagle_step_update_slot_mapping_and_metadata(
out_slot_mapping: [input_batch_size] output buffer for slot mapping
input_batch_size: total batch size including cudagraph padding;
defaults to batch_size (no padding)
"""
batch_size = positions_1d.shape[0]
if input_batch_size is None:
@@ -197,8 +196,7 @@ def eagle_prepare_inputs_padded_kernel(
num_rejected_tokens_gpu_ptr, # [num_reqs] (output)
num_reqs, # tl.int32
):
"""
Fused kernel for Eagle prepare_input_padded. This kernel computes the
"""Fused kernel for Eagle prepare_input_padded. This kernel computes the
token index to sample for each request, taking into account the number
of draft tokens and the number of valid sampled tokens (which is one more than
the number of accepted tokens).
@@ -270,8 +268,7 @@ def eagle_prepare_next_token_padded_kernel(
stride_sampled_token_ids, # tl.int32 (stride for dim 0)
BLOCK_SIZE_TOKENS: tl.constexpr, # Power-of-2 >= num_sampled_tokens_per_req
):
"""
Fused kernel for Eagle prepare_next_token_ids_padded. This kernel computes the
"""Fused kernel for Eagle prepare_next_token_ids_padded. This kernel computes the
number of valid (1 + accepted) tokens for each request, and the corresponding
"next" token id to sample from during speculative decoding. This is the
"last accepted token" from the sampled tokens, or the backup token if no
@@ -401,8 +398,7 @@ def extend_all_queries_by_N(
arange: torch.Tensor,
new_slot_mapping: torch.Tensor,
) -> CommonAttentionMetadata:
"""
Creates a new CommonAttentionMetadata with all query lengths increased by N.
"""Creates a new CommonAttentionMetadata with all query lengths increased by N.
Also all seq lens are increased by N.
This is useful e.g. in speculative decoding with parallel drafting, where we
extend each sequence by N tokens and predict all tokens in one pass.
@@ -453,8 +449,7 @@ def copy_and_expand_eagle_inputs_kernel(
shift_input_ids, # tl.bool
BLOCK_SIZE_TOKENS: tl.constexpr, # Blocks along token dim to handle prefills
):
"""
Copy and expand inputs from the target model to the drafting buffers for Eagle
"""Copy and expand inputs from the target model to the drafting buffers for Eagle
speculative decoding. This kernel handles padding slots and parallel drafting
tokens, if enabled.
"""
@@ -668,8 +663,7 @@ def copy_and_expand_dflash_inputs_kernel(
BLOCK_SIZE: tl.constexpr,
HAS_NUM_REJECTED: tl.constexpr = False,
):
"""
Fused kernel for DFlash first-pass input setup.
"""Fused kernel for DFlash first-pass input setup.
Per request, this kernel:
1. Copies context positions from target_positions to
@@ -853,5 +847,6 @@ def update_num_computed_tokens_for_batch_change(
def unconditional_to_conditional_rates(rates: list[float]) -> list[float]:
"""Convert per-position unconditional rates to per-position conditional
rates for the early-terminating rejection loop (c_i = p_i / p_{i-1})."""
rates for the early-terminating rejection loop (c_i = p_i / p_{i-1}).
"""
return [p / q if q > 0.0 else 0.0 for p, q in zip(rates, [1.0, *rates[:-1]])]