Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -219,8 +219,7 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
||||
is_graph_capturing: bool = False,
|
||||
slot_mappings: dict[str, torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Key differences to default dummy_run:
|
||||
"""Key differences to default dummy_run:
|
||||
- Only one forward pass due to parallel drafting
|
||||
- DFlash uses context states as unpadded metadata, so hidden_states will
|
||||
use the unpadded num_tokens instead of num_input_tokens
|
||||
|
||||
@@ -106,6 +106,7 @@ class ExtractHiddenStatesProposer:
|
||||
The main purpose is to cache hidden states, not to speculate.
|
||||
|
||||
Args:
|
||||
num_speculative_tokens: Number of draft tokens to return per request
|
||||
sampled_token_ids: Sampled token IDs from the target model
|
||||
target_hidden_states: List of hidden state tensors from target model
|
||||
(one per aux hidden state layer)
|
||||
@@ -117,6 +118,7 @@ class ExtractHiddenStatesProposer:
|
||||
Tuple of:
|
||||
- Draft tokens matching sampled tokens, shape [batch_size, 1]
|
||||
- KV connector output (if KV transfer is active), else None
|
||||
|
||||
"""
|
||||
assert num_speculative_tokens == self.num_speculative_tokens
|
||||
assert self.model is not None and isinstance(target_hidden_states, list)
|
||||
@@ -318,14 +320,12 @@ class ExtractHiddenStatesProposer:
|
||||
gpu_input_batch: InputBatch,
|
||||
discard_request_mask: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Prepare next token IDs for speculative decoding.
|
||||
"""Prepare next token IDs for speculative decoding.
|
||||
|
||||
Since num_speculative_tokens == 1, sampled_token_ids has shape
|
||||
(batch_size, 1). For each request we either use the sampled token
|
||||
(if valid and not discarded) or a backup token from the request state.
|
||||
"""
|
||||
|
||||
# Precompute backup token IDs for discarded requests.
|
||||
num_reqs = gpu_input_batch.num_reqs
|
||||
for i in range(num_reqs):
|
||||
@@ -359,6 +359,7 @@ class ExtractHiddenStatesProposer:
|
||||
Args:
|
||||
target_model: The target model (passed for compatibility with
|
||||
EagleProposer interface, but not used here)
|
||||
|
||||
"""
|
||||
# Get the target model's attention layers before loading draft model
|
||||
target_attn_layer_names = set(
|
||||
@@ -395,7 +396,8 @@ class ExtractHiddenStatesProposer:
|
||||
|
||||
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Validate all drafting layers belong to the same KV cache group
|
||||
and record the group index for common_attn_metadata selection."""
|
||||
and record the group index for common_attn_metadata selection.
|
||||
"""
|
||||
assert len(self.attn_layer_names) == 1
|
||||
layer = self.attn_layer_names[0]
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
|
||||
@@ -221,7 +221,8 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Draft layers span multiple KV cache groups (sliding + full
|
||||
attention with different head dimensions), so skip the base
|
||||
class single-group assertion."""
|
||||
class single-group assertion.
|
||||
"""
|
||||
|
||||
def initialize_attn_backend(
|
||||
self,
|
||||
@@ -229,7 +230,8 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
kernel_block_sizes: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Create separate AttentionGroup objects per KV cache spec
|
||||
so that each head-dim variant gets its own metadata builder."""
|
||||
so that each head-dim variant gets its own metadata builder.
|
||||
"""
|
||||
all_attn_layers = get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
|
||||
@@ -785,7 +785,8 @@ class SpecDecodeBaseProposer:
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Update positions, slot mappings, and sequence metadata for the
|
||||
next draft step. Returns the updated positions tensor."""
|
||||
next draft step. Returns the updated positions tensor.
|
||||
"""
|
||||
positions_1d = positions[0] if self.uses_mrope else positions
|
||||
if self.uses_mrope:
|
||||
out_pos = self.mrope_positions[0, :batch_size]
|
||||
@@ -1024,8 +1025,7 @@ class SpecDecodeBaseProposer:
|
||||
gpu_input_batch: InputBatch,
|
||||
num_scheduled_tokens: dict[str, int],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function is used to prepare the inputs for speculative decoding.
|
||||
"""This function is used to prepare the inputs for speculative decoding.
|
||||
It calculates the next token ids for each request based on the sampled
|
||||
token ids from the CPU. If a request has no sampled token ids (e.g.,
|
||||
during the initial decoding steps), it falls back to using the request
|
||||
@@ -1057,8 +1057,7 @@ class SpecDecodeBaseProposer:
|
||||
gpu_input_batch: InputBatch,
|
||||
discard_request_mask: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
This function is used to prepare the inputs for speculative decoding.
|
||||
"""This function is used to prepare the inputs for speculative decoding.
|
||||
It calculates the next token ids and the number of valid sampled tokens
|
||||
for each request, considering the "discarded" requests whose next token
|
||||
is not sampled and comes from `request.get_token_id()` instead. This is denoted
|
||||
@@ -1107,8 +1106,7 @@ class SpecDecodeBaseProposer:
|
||||
spec_decode_metadata: SpecDecodeMetadata,
|
||||
valid_sampled_tokens_count: torch.Tensor,
|
||||
) -> tuple[CommonAttentionMetadata, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
This function is used to prepare the inputs for speculative decoding
|
||||
"""This function is used to prepare the inputs for speculative decoding
|
||||
It updates the common_attn_metadata for speculative decoding,
|
||||
but does not consider the rejected tokens. Instead, all tokens
|
||||
are included as inputs to the speculator, with the rejected tokens
|
||||
@@ -1166,8 +1164,7 @@ class SpecDecodeBaseProposer:
|
||||
sampled_token_ids: list[list[int]],
|
||||
num_draft_tokens: list[int],
|
||||
) -> tuple[CommonAttentionMetadata, torch.Tensor]:
|
||||
"""
|
||||
This function is used to prepare the inputs for speculative decoding.
|
||||
"""This function is used to prepare the inputs for speculative decoding.
|
||||
It updates to the common_attn_metadata to account for the rejected
|
||||
tokens (and newly sampled tokens). It also returns the token indices
|
||||
of the tokens that should be fed to the speculator.
|
||||
@@ -1309,8 +1306,7 @@ class SpecDecodeBaseProposer:
|
||||
return base
|
||||
|
||||
def _get_model(self) -> nn.Module:
|
||||
"""
|
||||
Default method to call get_model(). Can be overridden by subclasses which
|
||||
"""Default method to call get_model(). Can be overridden by subclasses which
|
||||
need to customize model loading.
|
||||
"""
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
@@ -1422,8 +1418,7 @@ class SpecDecodeBaseProposer:
|
||||
self.parallel_drafting_hidden_state_tensor.copy_(flat_mask)
|
||||
|
||||
def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None:
|
||||
"""
|
||||
Some draft models may not have their own embedding layers, and some may
|
||||
"""Some draft models may not have their own embedding layers, and some may
|
||||
have a duplicate copy of the target model's embedding layers. In these cases,
|
||||
we share the target model's embedding layers with the draft model to save
|
||||
memory.
|
||||
@@ -1515,8 +1510,7 @@ class SpecDecodeBaseProposer:
|
||||
)
|
||||
|
||||
def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None:
|
||||
"""
|
||||
Some draft models may not have their own LM head, and some may have a
|
||||
"""Some draft models may not have their own LM head, and some may have a
|
||||
duplicate copy of the target model's LM head. In these cases, we share
|
||||
the target model's LM head with the draft model to save memory.
|
||||
"""
|
||||
@@ -1680,8 +1674,7 @@ class SpecDecodeBaseProposer:
|
||||
self.model(**kwargs)
|
||||
|
||||
def _get_eagle3_use_aux_hidden_state_from_config(self) -> bool:
|
||||
"""
|
||||
Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary
|
||||
"""Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary
|
||||
hidden states and directly uses the last layer output just like eagle1.
|
||||
They might indicate this by setting "use_aux_hidden_state" to False
|
||||
inside the "eagle_config" dict of their hf_config.
|
||||
@@ -1696,8 +1689,7 @@ class SpecDecodeBaseProposer:
|
||||
return use_aux_hidden_state
|
||||
|
||||
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""
|
||||
Validate that all drafting layers belong to the same KVCacheGroup.
|
||||
"""Validate that all drafting layers belong to the same KVCacheGroup.
|
||||
Need this assumption to ensure all drafting layers can use the
|
||||
same AttentionMetadata.
|
||||
May extend to multiple AttentionMetadata in the future.
|
||||
@@ -1723,8 +1715,7 @@ class SpecDecodeBaseProposer:
|
||||
kv_cache_config: KVCacheConfig,
|
||||
kernel_block_sizes: list[int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize AttentionGroups for draft layers using kv_cache_config.
|
||||
"""Initialize AttentionGroups for draft layers using kv_cache_config.
|
||||
Called from the model runner's initialize_metadata_builders.
|
||||
"""
|
||||
all_attn_layers = get_layers_from_vllm_config(
|
||||
|
||||
@@ -16,9 +16,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MedusaProposer:
|
||||
"""
|
||||
Medusa proposer class for generating token sequences
|
||||
"""
|
||||
"""Medusa proposer class for generating token sequences."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -72,6 +72,8 @@ class NgramProposer:
|
||||
"""Batch version of ngram proposer using numba for acceleration.
|
||||
|
||||
Args:
|
||||
num_requests:
|
||||
Number of requests in the batch.
|
||||
valid_ngram_requests:
|
||||
Set of indices of requests that need ngram proposals.
|
||||
num_tokens_no_spec:
|
||||
@@ -87,6 +89,7 @@ class NgramProposer:
|
||||
list[list[int]]:
|
||||
A list where each element is a list of proposed
|
||||
token IDs for the corresponding request.
|
||||
|
||||
"""
|
||||
draft_token_ids: list[list[int]] = []
|
||||
|
||||
@@ -211,8 +214,7 @@ def _find_longest_matched_ngram_and_propose_tokens(
|
||||
max_model_len: int,
|
||||
k: int,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Find the longest n-gram which matches the suffix of the given tokens
|
||||
"""Find the longest n-gram which matches the suffix of the given tokens
|
||||
whose length is within [min_ngram, max_ngram] (inclusive).
|
||||
|
||||
If found, we will extract k right after the matched ngram.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
GPU-accelerated N-gram proposer using fully async PyTorch tensor operations.
|
||||
"""GPU-accelerated N-gram proposer using fully async PyTorch tensor operations.
|
||||
|
||||
This version uses a fully vectorized approach with unfold and argmax for
|
||||
finding the first match across all sequences in parallel.
|
||||
@@ -52,8 +51,7 @@ class NgramGPUKernel(nn.Module):
|
||||
max_ngram_len: int,
|
||||
num_draft_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Find suffix n-gram matches and extract following tokens.
|
||||
"""Find suffix n-gram matches and extract following tokens.
|
||||
Searches for the earliest prior occurrence of the trailing n-gram,
|
||||
tries multiple lengths, and picks the longest valid match.
|
||||
|
||||
@@ -66,6 +64,7 @@ class NgramGPUKernel(nn.Module):
|
||||
|
||||
Returns:
|
||||
Draft token predictions; -1 means invalid/no match.
|
||||
|
||||
"""
|
||||
batch_size = token_ids.shape[0]
|
||||
max_seq_len = token_ids.shape[1]
|
||||
@@ -165,8 +164,7 @@ class NgramGPUKernel(nn.Module):
|
||||
token_ids_gpu: torch.Tensor,
|
||||
combined_mask: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for N-gram proposal using GPU tensor operations.
|
||||
"""Forward pass for N-gram proposal using GPU tensor operations.
|
||||
|
||||
Args:
|
||||
num_tokens_no_spec: Number of tokens for each sequence [batch_size]
|
||||
@@ -177,8 +175,8 @@ class NgramGPUKernel(nn.Module):
|
||||
draft_tokens: [batch_size, k] on GPU
|
||||
num_valid_draft_tokens: [batch_size] int32 on GPU, count of
|
||||
leading valid (non -1) tokens per request.
|
||||
"""
|
||||
|
||||
"""
|
||||
device = token_ids_gpu.device
|
||||
|
||||
# Infer batch size to preserve dynamic shape.
|
||||
@@ -282,8 +280,7 @@ class NgramProposerGPU:
|
||||
pattern_len: int,
|
||||
device: str = "cuda",
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Generate random test data with n-gram repetitions.
|
||||
"""Generate random test data with n-gram repetitions.
|
||||
|
||||
Args:
|
||||
batch_size: Number of sequences in the batch
|
||||
@@ -296,6 +293,7 @@ class NgramProposerGPU:
|
||||
num_tokens: [batch_size] tensor
|
||||
sampled_flags: [batch_size] bool tensor
|
||||
valid_mask: [batch_size] bool tensor
|
||||
|
||||
"""
|
||||
token_ids = torch.zeros(
|
||||
batch_size,
|
||||
@@ -321,8 +319,7 @@ class NgramProposerGPU:
|
||||
valid_sampled_token_ids_gpu: torch.Tensor, # [batch_size, num_spec_tokens + 1]
|
||||
valid_sampled_tokens_count: torch.Tensor, # [batch_size]
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Propose draft tokens using GPU-accelerated n-gram matching.
|
||||
"""Propose draft tokens using GPU-accelerated n-gram matching.
|
||||
|
||||
Scatter sampled tokens into `token_ids_gpu`, compute temporary
|
||||
updated lengths, then run the kernel.
|
||||
@@ -338,6 +335,7 @@ class NgramProposerGPU:
|
||||
draft_tokens: Proposed draft token IDs [batch_size, k]
|
||||
num_valid_draft_tokens: Count of leading valid draft tokens
|
||||
per request [batch_size]
|
||||
|
||||
"""
|
||||
assert num_speculative_tokens == self.k
|
||||
assert token_ids_gpu.device == self.device
|
||||
@@ -398,8 +396,7 @@ class NgramProposerGPU:
|
||||
num_tokens_no_spec: torch.Tensor,
|
||||
discard_request_mask: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Prepare speculative decoding inputs on device:
|
||||
"""Prepare speculative decoding inputs on device:
|
||||
compute next token ids and valid counts, honoring discarded requests
|
||||
and rejected tokens, without CPU-GPU sync.
|
||||
"""
|
||||
@@ -485,6 +482,7 @@ def update_scheduler_for_invalid_drafts(
|
||||
num_valid_draft_tokens_cpu: CPU buffer of valid draft counts.
|
||||
scheduler_output: Scheduler metadata to update in-place.
|
||||
req_id_to_index: Request-id to batch-index mapping.
|
||||
|
||||
"""
|
||||
req_data = scheduler_output.scheduled_cached_reqs
|
||||
num_valid_draft_tokens_event.synchronize()
|
||||
@@ -652,9 +650,7 @@ def copy_num_valid_draft_tokens(
|
||||
num_valid_draft_tokens: torch.Tensor | None,
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
"""
|
||||
Async D2H copy of per-request valid draft counts.
|
||||
"""
|
||||
"""Async D2H copy of per-request valid draft counts."""
|
||||
if num_valid_draft_tokens is None:
|
||||
return
|
||||
|
||||
|
||||
@@ -155,7 +155,6 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
|
||||
def _maybe_share_lm_head(self, target_language_model: torch.nn.Module) -> None:
|
||||
"""Step3.5 MTP uses the lm_head stored in each MTP layer."""
|
||||
|
||||
# The base MTP path shares target lm_head into shared_head.head.
|
||||
# Step3.5 checkpoints carry per-MTP-layer shared_head weights.
|
||||
return
|
||||
|
||||
@@ -7,8 +7,7 @@ from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
|
||||
|
||||
class SuffixDecodingProposer:
|
||||
"""
|
||||
Speculative decoding proposer for Suffix Decoding (https://arxiv.org/pdf/2411.04975).
|
||||
"""Speculative decoding proposer for Suffix Decoding (https://arxiv.org/pdf/2411.04975).
|
||||
This class imports and uses the official implementation from Arctic Inference
|
||||
(https://github.com/snowflakedb/ArcticInference).
|
||||
"""
|
||||
|
||||
@@ -50,8 +50,7 @@ def eagle_step_slot_mapping_metadata_kernel(
|
||||
PAD_ID: tl.constexpr,
|
||||
batch_size,
|
||||
):
|
||||
"""
|
||||
Fused kernel for EAGLE autoregressive step: updates positions, slot mapping,
|
||||
"""Fused kernel for EAGLE autoregressive step: updates positions, slot mapping,
|
||||
and sequence lengths in a single kernel to reduce launch overhead.
|
||||
|
||||
Launched with input_batch_size threads. Threads with req_idx >= batch_size
|
||||
@@ -149,8 +148,7 @@ def eagle_step_update_slot_mapping_and_metadata(
|
||||
out_slot_mapping: torch.Tensor,
|
||||
input_batch_size: int | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Fused update of slot mapping and metadata for one EAGLE autoregressive step.
|
||||
"""Fused update of slot mapping and metadata for one EAGLE autoregressive step.
|
||||
Updates seq_lens in place. Writes to out_clamped_positions and out_slot_mapping.
|
||||
|
||||
When input_batch_size > batch_size, threads beyond batch_size write
|
||||
@@ -166,6 +164,7 @@ def eagle_step_update_slot_mapping_and_metadata(
|
||||
out_slot_mapping: [input_batch_size] output buffer for slot mapping
|
||||
input_batch_size: total batch size including cudagraph padding;
|
||||
defaults to batch_size (no padding)
|
||||
|
||||
"""
|
||||
batch_size = positions_1d.shape[0]
|
||||
if input_batch_size is None:
|
||||
@@ -197,8 +196,7 @@ def eagle_prepare_inputs_padded_kernel(
|
||||
num_rejected_tokens_gpu_ptr, # [num_reqs] (output)
|
||||
num_reqs, # tl.int32
|
||||
):
|
||||
"""
|
||||
Fused kernel for Eagle prepare_input_padded. This kernel computes the
|
||||
"""Fused kernel for Eagle prepare_input_padded. This kernel computes the
|
||||
token index to sample for each request, taking into account the number
|
||||
of draft tokens and the number of valid sampled tokens (which is one more than
|
||||
the number of accepted tokens).
|
||||
@@ -270,8 +268,7 @@ def eagle_prepare_next_token_padded_kernel(
|
||||
stride_sampled_token_ids, # tl.int32 (stride for dim 0)
|
||||
BLOCK_SIZE_TOKENS: tl.constexpr, # Power-of-2 >= num_sampled_tokens_per_req
|
||||
):
|
||||
"""
|
||||
Fused kernel for Eagle prepare_next_token_ids_padded. This kernel computes the
|
||||
"""Fused kernel for Eagle prepare_next_token_ids_padded. This kernel computes the
|
||||
number of valid (1 + accepted) tokens for each request, and the corresponding
|
||||
"next" token id to sample from during speculative decoding. This is the
|
||||
"last accepted token" from the sampled tokens, or the backup token if no
|
||||
@@ -401,8 +398,7 @@ def extend_all_queries_by_N(
|
||||
arange: torch.Tensor,
|
||||
new_slot_mapping: torch.Tensor,
|
||||
) -> CommonAttentionMetadata:
|
||||
"""
|
||||
Creates a new CommonAttentionMetadata with all query lengths increased by N.
|
||||
"""Creates a new CommonAttentionMetadata with all query lengths increased by N.
|
||||
Also all seq lens are increased by N.
|
||||
This is useful e.g. in speculative decoding with parallel drafting, where we
|
||||
extend each sequence by N tokens and predict all tokens in one pass.
|
||||
@@ -453,8 +449,7 @@ def copy_and_expand_eagle_inputs_kernel(
|
||||
shift_input_ids, # tl.bool
|
||||
BLOCK_SIZE_TOKENS: tl.constexpr, # Blocks along token dim to handle prefills
|
||||
):
|
||||
"""
|
||||
Copy and expand inputs from the target model to the drafting buffers for Eagle
|
||||
"""Copy and expand inputs from the target model to the drafting buffers for Eagle
|
||||
speculative decoding. This kernel handles padding slots and parallel drafting
|
||||
tokens, if enabled.
|
||||
"""
|
||||
@@ -668,8 +663,7 @@ def copy_and_expand_dflash_inputs_kernel(
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_NUM_REJECTED: tl.constexpr = False,
|
||||
):
|
||||
"""
|
||||
Fused kernel for DFlash first-pass input setup.
|
||||
"""Fused kernel for DFlash first-pass input setup.
|
||||
|
||||
Per request, this kernel:
|
||||
1. Copies context positions from target_positions to
|
||||
@@ -853,5 +847,6 @@ def update_num_computed_tokens_for_batch_change(
|
||||
|
||||
def unconditional_to_conditional_rates(rates: list[float]) -> list[float]:
|
||||
"""Convert per-position unconditional rates to per-position conditional
|
||||
rates for the early-terminating rejection loop (c_i = p_i / p_{i-1})."""
|
||||
rates for the early-terminating rejection loop (c_i = p_i / p_{i-1}).
|
||||
"""
|
||||
return [p / q if q > 0.0 else 0.0 for p, q in zip(rates, [1.0, *rates[:-1]])]
|
||||
|
||||
Reference in New Issue
Block a user