Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -55,8 +55,7 @@ BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [
|
||||
|
||||
|
||||
def _load_logitsprocs_plugins() -> list[type[LogitsProcessor]]:
|
||||
"""Load all installed logit processor plugins"""
|
||||
|
||||
"""Load all installed logit processor plugins."""
|
||||
from importlib.metadata import entry_points
|
||||
|
||||
installed_logitsprocs_plugins = entry_points(group=LOGITSPROCS_GROUP)
|
||||
@@ -171,6 +170,7 @@ def _load_custom_logitsprocs(
|
||||
|
||||
Returns:
|
||||
A list of all loaded logitproc types
|
||||
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -238,7 +238,7 @@ def validate_logits_processors_parameters(
|
||||
|
||||
|
||||
class AdapterLogitsProcessor(LogitsProcessor):
|
||||
"""Wrapper for per-request logits processors
|
||||
"""Wrapper for per-request logits processors.
|
||||
|
||||
To wrap a specific per-request logits processor,
|
||||
* Subclass `AdapterLogitsProcessor`
|
||||
@@ -264,7 +264,6 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
these arguments are used, the vLLM logits processor interface requires
|
||||
all three arguments to be present.
|
||||
"""
|
||||
|
||||
# Map req index -> logits processor state
|
||||
#
|
||||
# State representation is a partial[Tensor] comprising a request-level
|
||||
@@ -301,7 +300,7 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
prompt_ids: list[int] | None,
|
||||
output_ids: list[int],
|
||||
) -> partial[torch.Tensor] | None:
|
||||
"""Return state representation for new request
|
||||
"""Return state representation for new request.
|
||||
|
||||
Returns None if logits processor is not applicable to request
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ class MinPLogitsProcessor(LogitsProcessor):
|
||||
self.min_p: torch.Tensor = self.min_p_device[:0]
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
"""Min-p never impacts greedy sampling"""
|
||||
"""Min-p never impacts greedy sampling."""
|
||||
return True
|
||||
|
||||
def get_min_p_by_index(self, index: int) -> float:
|
||||
@@ -129,7 +129,8 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
"""Logit bias can rebalance token probabilities and change the
|
||||
outcome of argmax in greedy sampling."""
|
||||
outcome of argmax in greedy sampling.
|
||||
"""
|
||||
return False
|
||||
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
@@ -188,7 +189,8 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
"""By censoring stop tokens, min-tokens can change the outcome
|
||||
of the argmax operation in greedy sampling."""
|
||||
of the argmax operation in greedy sampling.
|
||||
"""
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
@@ -377,7 +379,6 @@ def process_dict_updates(
|
||||
new_state: Callable[[SamplingParams, list[int] | None, list[int]], T | None],
|
||||
) -> bool:
|
||||
"""Utility function to update dict state for sparse LogitsProcessors."""
|
||||
|
||||
if not batch_update:
|
||||
# Nothing to do.
|
||||
return False
|
||||
|
||||
@@ -35,7 +35,7 @@ MovedRequest = tuple[int, int, MoveDirectionality]
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchUpdate:
|
||||
"""Persistent batch state change info for logitsprocs"""
|
||||
"""Persistent batch state change info for logitsprocs."""
|
||||
|
||||
batch_size: int # Current num reqs in batch
|
||||
|
||||
@@ -104,5 +104,6 @@ class LogitsProcessor(ABC):
|
||||
Args:
|
||||
batch_update: Non-None iff there have been changes
|
||||
to the batch makeup.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -69,7 +69,8 @@ class BatchUpdateBuilder:
|
||||
@property
|
||||
def removed(self) -> list[RemovedRequest]:
|
||||
"""Removed request indices sorted in
|
||||
descending order"""
|
||||
descending order
|
||||
"""
|
||||
self._ensure_removed_sorted()
|
||||
return self._removed
|
||||
|
||||
@@ -81,6 +82,7 @@ class BatchUpdateBuilder:
|
||||
|
||||
Args:
|
||||
index: request index
|
||||
|
||||
"""
|
||||
if self._is_removed_sorted:
|
||||
raise RuntimeError(
|
||||
@@ -93,14 +95,14 @@ class BatchUpdateBuilder:
|
||||
return bool(self._removed)
|
||||
|
||||
def peek_removed(self) -> int | None:
|
||||
"""Return lowest removed request index"""
|
||||
"""Return lowest removed request index."""
|
||||
if self.has_removed():
|
||||
self._ensure_removed_sorted()
|
||||
return self._removed[-1]
|
||||
return None
|
||||
|
||||
def pop_removed(self) -> int | None:
|
||||
"""Pop lowest removed request index"""
|
||||
"""Pop lowest removed request index."""
|
||||
if self.has_removed():
|
||||
self._ensure_removed_sorted()
|
||||
return self._removed.pop()
|
||||
@@ -125,6 +127,7 @@ class BatchUpdateBuilder:
|
||||
|
||||
Returns:
|
||||
Frozen logitsprocs batch update instance; `None` if no updates
|
||||
|
||||
"""
|
||||
# Reset removal-sorting logic
|
||||
self._is_removed_sorted = False
|
||||
|
||||
@@ -9,8 +9,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
@torch.compile(backend=current_platform.simple_compile_backend)
|
||||
def batched_count_greater_than(x: torch.Tensor, values: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Counts elements in each row of x that are greater than the corresponding
|
||||
"""Counts elements in each row of x that are greater than the corresponding
|
||||
value in values. Use torch.compile to generate an optimized kernel for
|
||||
this function. otherwise, it will create additional copies of the input
|
||||
tensors and cause memory issues.
|
||||
@@ -21,6 +20,7 @@ def batched_count_greater_than(x: torch.Tensor, values: torch.Tensor) -> torch.T
|
||||
|
||||
Returns:
|
||||
torch.Tensor: A 1D tensor of shape (batch_size,) with the counts.
|
||||
|
||||
"""
|
||||
torch._check(x.shape[0] >= 1)
|
||||
torch._check(x.shape[0] == values.shape[0])
|
||||
|
||||
@@ -15,9 +15,7 @@ def apply_all_penalties(
|
||||
repetition_penalties: torch.Tensor,
|
||||
output_token_ids: list[list[int]],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Applies presence, frequency and repetition penalties to the logits.
|
||||
"""
|
||||
"""Applies presence, frequency and repetition penalties to the logits."""
|
||||
_, vocab_size = logits.shape
|
||||
output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size, logits.device)
|
||||
|
||||
@@ -41,9 +39,7 @@ def apply_all_penalties(
|
||||
def _convert_to_tensors(
|
||||
output_token_ids: list[list[int]], vocab_size: int, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Convert the different list data structures to tensors.
|
||||
"""
|
||||
"""Convert the different list data structures to tensors."""
|
||||
output_tokens_tensor = make_tensor_with_pad(
|
||||
output_token_ids,
|
||||
# Use the value of vocab_size as a pad since we don't have a
|
||||
|
||||
@@ -91,8 +91,7 @@ def flashinfer_sampler_supported() -> bool:
|
||||
|
||||
|
||||
class TopKTopPSampler(nn.Module):
|
||||
"""
|
||||
Module that performs optional top-k and top-p filtering followed by
|
||||
"""Module that performs optional top-k and top-p filtering followed by
|
||||
weighted random sampling of logits.
|
||||
|
||||
Implementations may update the logits tensor in-place.
|
||||
@@ -154,8 +153,7 @@ class TopKTopPSampler(nn.Module):
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
PyTorch-native implementation of top-k and top-p sampling.
|
||||
"""PyTorch-native implementation of top-k and top-p sampling.
|
||||
|
||||
The logits tensor may be updated in-place.
|
||||
"""
|
||||
@@ -207,8 +205,7 @@ class TopKTopPSampler(nn.Module):
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
PyTorch-native implementation of top-k and top-p sampling for CPU.
|
||||
"""PyTorch-native implementation of top-k and top-p sampling for CPU.
|
||||
|
||||
The logits tensor may be updated in-place.
|
||||
"""
|
||||
@@ -423,8 +420,7 @@ def apply_top_k_top_p_pytorch(
|
||||
|
||||
|
||||
def apply_top_k_only(logits: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Apply top-k mask to the logits.
|
||||
"""Apply top-k mask to the logits.
|
||||
|
||||
This implementation doesn't involve sorting the entire vocab.
|
||||
Note however that it involves a GPU->CPU sync which can be detrimental for
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Combined Top-K and Top-P Triton kernels.
|
||||
"""Combined Top-K and Top-P Triton kernels.
|
||||
|
||||
Based on the paper "Qrita: High-performance Top-k and Top-p Algorithm for GPUs
|
||||
using Pivot-based Truncation and Selection" By Park et al.
|
||||
@@ -1617,8 +1616,7 @@ def apply_top_k_top_p_triton(
|
||||
p: torch.Tensor | None,
|
||||
mask_value: float = float("-inf"),
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply combined top-k and top-p masking using Triton.
|
||||
"""Apply combined top-k and top-p masking using Triton.
|
||||
|
||||
Top-k is applied first (by logit value), then top-p is applied
|
||||
to the remaining k values (by probability).
|
||||
@@ -1634,6 +1632,7 @@ def apply_top_k_top_p_triton(
|
||||
|
||||
Returns:
|
||||
The masked logits tensor. It may or may not be modified in-place.
|
||||
|
||||
"""
|
||||
assert logits.ndim == 2
|
||||
assert logits.dtype == torch.float32
|
||||
|
||||
@@ -43,8 +43,7 @@ MAX_SPEC_LEN = 128
|
||||
|
||||
|
||||
class RejectionSampler(nn.Module):
|
||||
"""
|
||||
The implementation strictly follows the algorithm described in
|
||||
"""The implementation strictly follows the algorithm described in
|
||||
https://arxiv.org/abs/2211.17192.
|
||||
However, we want to clarify the terminology used in the implementation:
|
||||
accepted tokens: tokens that are accepted based on the relationship
|
||||
@@ -113,8 +112,7 @@ class RejectionSampler(nn.Module):
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> SamplerOutput:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
metadata:
|
||||
Metadata for spec decoding.
|
||||
draft_probs (Optional[torch.Tensor]):
|
||||
@@ -130,10 +128,12 @@ class RejectionSampler(nn.Module):
|
||||
sampling_metadata (vllm.v1.sample.metadata.SamplingMetadata):
|
||||
Additional metadata needed for sampling, such as temperature,
|
||||
top-k/top-p parameters, or other relevant information.
|
||||
|
||||
Returns:
|
||||
SamplerOutput:
|
||||
Contains the final output token IDs and their logprobs if
|
||||
requested.
|
||||
|
||||
"""
|
||||
assert metadata.max_spec_len <= MAX_SPEC_LEN
|
||||
|
||||
@@ -272,6 +272,7 @@ class RejectionSampler(nn.Module):
|
||||
logprobs_tensors: LogprobsTensors | None = None,
|
||||
) -> tuple[list[list[int]], LogprobsLists | None]:
|
||||
"""Parse the output of the rejection sampler.
|
||||
|
||||
Args:
|
||||
output_token_ids: The sampled token IDs in shape
|
||||
[batch_size, max_spec_len + 1]. The rejected tokens are
|
||||
@@ -280,8 +281,10 @@ class RejectionSampler(nn.Module):
|
||||
vocab_size: The size of the vocabulary.
|
||||
discard_req_indices: Optional row indices to discard tokens in.
|
||||
logprobs_tensors: Optional logprobs tensors to filter.
|
||||
|
||||
Returns:
|
||||
A list of lists of token IDs.
|
||||
|
||||
"""
|
||||
output_token_ids_np = output_token_ids.cpu().numpy()
|
||||
# Create mask for valid tokens.
|
||||
@@ -536,6 +539,7 @@ def apply_sampling_constraints(
|
||||
Returns:
|
||||
torch.Tensor: Processed logits if non-greedy sampling is used,
|
||||
otherwise returns the original logits.
|
||||
|
||||
"""
|
||||
assert logits.ndim == 2
|
||||
assert cu_num_draft_tokens.ndim == 1
|
||||
@@ -597,8 +601,10 @@ def expand_batch_to_tokens(
|
||||
Value to be replaced if it is found in x.
|
||||
replace_to: int = 0
|
||||
Value to replace with when replace_from is found.
|
||||
|
||||
Returns:
|
||||
expanded_x: [num_tokens] tensor.
|
||||
|
||||
"""
|
||||
batch_size = x.shape[0]
|
||||
assert cu_num_tokens.shape[0] == batch_size
|
||||
@@ -619,8 +625,7 @@ def generate_uniform_probs(
|
||||
generators: dict[int, torch.Generator],
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Generates a batch of uniform random samples, with optional seeding
|
||||
"""Generates a batch of uniform random samples, with optional seeding
|
||||
if available.
|
||||
|
||||
This method creates a tensor of shape `(num_tokens, )` filled
|
||||
@@ -639,10 +644,12 @@ def generate_uniform_probs(
|
||||
`torch.Generator` objects.
|
||||
device: torch.device
|
||||
The device on which to allocate the tensor.
|
||||
|
||||
Returns:
|
||||
uniform_rand: torch.Tensor
|
||||
A tensor of shape `(num_tokens, )` containing uniform
|
||||
random values in the range [0, 1).
|
||||
|
||||
"""
|
||||
# NOTE(woosuk): We deliberately use float64 instead of float32 here
|
||||
# because when using float32, there's a non-negligible chance that
|
||||
|
||||
@@ -19,8 +19,7 @@ _SAMPLING_EPS = 1e-5
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
"""
|
||||
A layer that samples the next tokens from the model's outputs
|
||||
"""A layer that samples the next tokens from the model's outputs
|
||||
with the following steps in order:
|
||||
|
||||
1. If logprobs are requested:
|
||||
@@ -170,6 +169,7 @@ class Sampler(nn.Module):
|
||||
Returns:
|
||||
LogprobsTensors with logprobs for the specified tokens, or None
|
||||
if no requests have logprob_token_ids.
|
||||
|
||||
"""
|
||||
if not logprob_token_ids:
|
||||
return None
|
||||
@@ -252,7 +252,6 @@ class Sampler(nn.Module):
|
||||
The various logits processing functions called in this method
|
||||
may update the logits tensor in-place.
|
||||
"""
|
||||
|
||||
logprobs_mode = logprobs_mode_override or self.logprobs_mode
|
||||
assert not (sampling_metadata.all_greedy and sampling_metadata.all_random)
|
||||
if sampling_metadata.all_random:
|
||||
@@ -312,8 +311,7 @@ class Sampler(nn.Module):
|
||||
num_logprobs: int,
|
||||
token_ids: torch.Tensor,
|
||||
) -> LogprobsTensors:
|
||||
"""
|
||||
Gather logprobs for topk and sampled/prompt token.
|
||||
"""Gather logprobs for topk and sampled/prompt token.
|
||||
|
||||
Args:
|
||||
logprobs: (num tokens) x (vocab) tensor
|
||||
@@ -329,6 +327,7 @@ class Sampler(nn.Module):
|
||||
Top-k int indices tensor, (num tokens) x (num_logprobs + 1)
|
||||
Top-k float logprobs tensor, (num tokens) x (num_logprobs + 1)
|
||||
Sampled token rank tensor, (num tokens)
|
||||
|
||||
"""
|
||||
assert token_ids.dtype == torch.int64
|
||||
# Find the topK values.
|
||||
|
||||
Reference in New Issue
Block a user