Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+66 -44
View File
@@ -13,15 +13,18 @@ import torch
from vllm.logger import init_logger
from vllm.logits_process import LogitsProcessor as RequestLogitsProcessor
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.logits_processor.builtin import (LogitBiasLogitsProcessor,
MinPLogitsProcessor,
MinTokensLogitsProcessor,
process_dict_updates)
from vllm.v1.sample.logits_processor.interface import (BatchUpdate,
LogitsProcessor,
MoveDirectionality)
from vllm.v1.sample.logits_processor.state import (BatchUpdateBuilder,
LogitsProcessors)
from vllm.v1.sample.logits_processor.builtin import (
LogitBiasLogitsProcessor,
MinPLogitsProcessor,
MinTokensLogitsProcessor,
process_dict_updates,
)
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
LogitsProcessor,
MoveDirectionality,
)
from vllm.v1.sample.logits_processor.state import BatchUpdateBuilder, LogitsProcessors
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -30,10 +33,11 @@ logger = init_logger(__name__)
# Error message when the user tries to initialize vLLM with a pooling model
# and custom logitsproces
STR_POOLING_REJECTS_LOGITSPROCS = ("Pooling models do not support custom"
" logits processors.")
STR_POOLING_REJECTS_LOGITSPROCS = (
"Pooling models do not support custom logits processors."
)
LOGITSPROCS_GROUP = 'vllm.logits_processors'
LOGITSPROCS_GROUP = "vllm.logits_processors"
BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [
MinTokensLogitsProcessor,
@@ -54,27 +58,29 @@ def _load_logitsprocs_plugins() -> list[type[LogitsProcessor]]:
installed_logitsprocs_plugins = entry_points(group=LOGITSPROCS_GROUP)
if len(installed_logitsprocs_plugins) == 0:
logger.debug("No logitsprocs plugins installed (group %s).",
LOGITSPROCS_GROUP)
logger.debug("No logitsprocs plugins installed (group %s).", LOGITSPROCS_GROUP)
return []
# Load logitsprocs plugins
logger.debug("Loading installed logitsprocs plugins (group %s):",
LOGITSPROCS_GROUP)
logger.debug("Loading installed logitsprocs plugins (group %s):", LOGITSPROCS_GROUP)
classes: list[type[LogitsProcessor]] = []
for entrypoint in installed_logitsprocs_plugins:
try:
logger.debug("- Loading logitproc plugin entrypoint=%s target=%s",
entrypoint.name, entrypoint.value)
logger.debug(
"- Loading logitproc plugin entrypoint=%s target=%s",
entrypoint.name,
entrypoint.value,
)
classes.append(entrypoint.load())
except Exception as e:
raise RuntimeError(
f"Failed to load LogitsProcessor plugin {entrypoint}") from e
f"Failed to load LogitsProcessor plugin {entrypoint}"
) from e
return classes
def _load_logitsprocs_by_fqcns(
logits_processors: Optional[Sequence[Union[str, type[LogitsProcessor]]]]
logits_processors: Optional[Sequence[Union[str, type[LogitsProcessor]]]],
) -> list[type[LogitsProcessor]]:
"""Load logit processor types, identifying them by fully-qualified class
names (FQCNs).
@@ -99,13 +105,14 @@ def _load_logitsprocs_by_fqcns(
logger.debug(
"%s additional custom logits processors specified, checking whether "
"they need to be loaded.", len(logits_processors))
"they need to be loaded.",
len(logits_processors),
)
classes: list[type[LogitsProcessor]] = []
for ldx, logitproc in enumerate(logits_processors):
if isinstance(logitproc, type):
logger.debug(" - Already-loaded logit processor: %s",
logitproc.__name__)
logger.debug(" - Already-loaded logit processor: %s", logitproc.__name__)
if not issubclass(logitproc, LogitsProcessor):
raise ValueError(
f"{logitproc.__name__} is not a subclass of LogitsProcessor"
@@ -131,8 +138,7 @@ def _load_logitsprocs_by_fqcns(
if not isinstance(obj, type):
raise ValueError("Loaded logit processor must be a type.")
if not issubclass(obj, LogitsProcessor):
raise ValueError(
f"{obj.__name__} must be a subclass of LogitsProcessor")
raise ValueError(f"{obj.__name__} must be a subclass of LogitsProcessor")
classes.append(obj)
return classes
@@ -155,13 +161,13 @@ def _load_custom_logitsprocs(
A list of all loaded logitproc types
"""
from vllm.platforms import current_platform
if current_platform.is_tpu():
# No logitsprocs specified by caller
# TODO(andy) - vLLM V1 on TPU does not support custom logitsprocs
return []
return (_load_logitsprocs_plugins() +
_load_logitsprocs_by_fqcns(logits_processors))
return _load_logitsprocs_plugins() + _load_logitsprocs_by_fqcns(logits_processors)
def build_logitsprocs(
@@ -174,23 +180,28 @@ def build_logitsprocs(
if is_pooling_model:
if custom_logitsprocs:
raise ValueError(STR_POOLING_REJECTS_LOGITSPROCS)
logger.debug("Skipping logits processor loading because pooling models"
" do not support logits processors.")
logger.debug(
"Skipping logits processor loading because pooling models"
" do not support logits processors."
)
return LogitsProcessors()
custom_logitsprocs_classes = _load_custom_logitsprocs(custom_logitsprocs)
return LogitsProcessors(
ctor(vllm_config, device, is_pin_memory) for ctor in itertools.chain(
BUILTIN_LOGITS_PROCESSORS, custom_logitsprocs_classes))
ctor(vllm_config, device, is_pin_memory)
for ctor in itertools.chain(
BUILTIN_LOGITS_PROCESSORS, custom_logitsprocs_classes
)
)
class AdapterLogitsProcessor(LogitsProcessor):
"""Wrapper for per-request logits processors
To wrap a specific per-request logits processor,
* Subclass `AdapterLogitsProcessor`
* Implement `self.is_argmax_invariant()` base-class method
* Implement `self.new_req_logits_processor(params)`
`self.__init__(vllm_config, device, is_pin_memory)` does not need to be
overridden in general. However, to implement custom constructor behavior -
especially any logic which operates on or stores `vllm_config`, `device`,
@@ -199,8 +210,9 @@ class AdapterLogitsProcessor(LogitsProcessor):
`super().__init__(vllm_config, device, is_pin_memory)`
"""
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
is_pin_memory: bool):
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
):
"""Subclass must invoke
`super().__init__(vllm_config, device, is_pin_memory)`.
@@ -236,7 +248,7 @@ class AdapterLogitsProcessor(LogitsProcessor):
Returns:
None if logits processor should not be applied to request; otherwise
returns a `RequestLogitsProcessor` instance
"""
raise NotImplementedError
@@ -257,11 +269,14 @@ class AdapterLogitsProcessor(LogitsProcessor):
Returns:
logits processor partial[Tensor] or None
"""
if req_lp := self.new_req_logits_processor(params):
args = [prompt_ids, output_ids] if (len(
inspect.signature(req_lp).parameters) == 3) else [output_ids]
args = (
[prompt_ids, output_ids]
if (len(inspect.signature(req_lp).parameters) == 3)
else [output_ids]
)
return partial(req_lp, *args)
return None
@@ -286,9 +301,16 @@ class AdapterLogitsProcessor(LogitsProcessor):
__all__ = [
"LogitsProcessor", "LogitBiasLogitsProcessor", "MinPLogitsProcessor",
"MinTokensLogitsProcessor", "BatchUpdate", "BatchUpdateBuilder",
"MoveDirectionality", "LogitsProcessors", "build_logitsprocs",
"STR_POOLING_REJECTS_LOGITSPROCS", "LOGITSPROCS_GROUP",
"AdapterLogitsProcessor"
"LogitsProcessor",
"LogitBiasLogitsProcessor",
"MinPLogitsProcessor",
"MinTokensLogitsProcessor",
"BatchUpdate",
"BatchUpdateBuilder",
"MoveDirectionality",
"LogitsProcessors",
"build_logitsprocs",
"STR_POOLING_REJECTS_LOGITSPROCS",
"LOGITSPROCS_GROUP",
"AdapterLogitsProcessor",
]
+57 -58
View File
@@ -6,9 +6,11 @@ from typing import TYPE_CHECKING, Callable, Optional, TypeVar
import torch
from vllm import SamplingParams
from vllm.v1.sample.logits_processor.interface import (BatchUpdate,
LogitsProcessor,
MoveDirectionality)
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
LogitsProcessor,
MoveDirectionality,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -17,25 +19,24 @@ T = TypeVar("T")
class MinPLogitsProcessor(LogitsProcessor):
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
is_pin_memory: bool):
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
):
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
self.min_p_count: int = 0
self.min_p_cpu_tensor = torch.zeros((max_num_reqs, ),
dtype=torch.float32,
device="cpu",
pin_memory=is_pin_memory)
self.min_p_cpu_tensor = torch.zeros(
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=is_pin_memory
)
self.min_p_cpu = self.min_p_cpu_tensor.numpy()
self.use_double_tensor = torch.device(device).type != "cpu"
if self.use_double_tensor:
# Pre-allocated device tensor
self.min_p_device: torch.Tensor = torch.empty((max_num_reqs, ),
dtype=torch.float32,
device=device)
self.min_p_device: torch.Tensor = torch.empty(
(max_num_reqs,), dtype=torch.float32, device=device
)
else:
self.min_p_device = self.min_p_cpu_tensor
# Current slice of the device tensor
@@ -93,8 +94,7 @@ class MinPLogitsProcessor(LogitsProcessor):
if self.min_p_count and (needs_update or self.min_p.shape[0] != size):
self.min_p = self.min_p_device[:size]
if self.use_double_tensor:
self.min_p.copy_(self.min_p_cpu_tensor[:size],
non_blocking=True)
self.min_p.copy_(self.min_p_cpu_tensor[:size], non_blocking=True)
self.min_p.unsqueeze_(1)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
@@ -104,28 +104,27 @@ class MinPLogitsProcessor(LogitsProcessor):
# Convert logits to probability distribution
probability_values = torch.nn.functional.softmax(logits, dim=-1)
# Calculate maximum probabilities per sequence
max_probabilities = torch.amax(probability_values,
dim=-1,
keepdim=True)
max_probabilities = torch.amax(probability_values, dim=-1, keepdim=True)
# Adjust min_p
adjusted_min_p = max_probabilities.mul_(self.min_p)
# Identify valid tokens using threshold comparison
invalid_token_mask = probability_values < adjusted_min_p
# Apply mask using boolean indexing
logits[invalid_token_mask] = -float('inf')
logits[invalid_token_mask] = -float("inf")
return logits
class LogitBiasLogitsProcessor(LogitsProcessor):
def __init__(self, _, device: torch.device, is_pin_memory: bool):
self.device = device
self.pin_memory = is_pin_memory
self.biases: dict[int, dict[int, float]] = {}
self.bias_tensor: torch.Tensor = torch.tensor(())
self.logits_slice = (self._device_tensor([], torch.int32),
self._device_tensor([], torch.int32))
self.logits_slice = (
self._device_tensor([], torch.int32),
self._device_tensor([], torch.int32),
)
def is_argmax_invariant(self) -> bool:
"""Logit bias can rebalance token probabilities and change the
@@ -134,8 +133,8 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
def update_state(self, batch_update: Optional[BatchUpdate]):
needs_update = process_dict_updates(
self.biases, batch_update,
lambda params, _, __: params.logit_bias or None)
self.biases, batch_update, lambda params, _, __: params.logit_bias or None
)
# Update tensors if needed.
if needs_update:
@@ -148,15 +147,15 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
biases.extend(lb.values())
self.bias_tensor = self._device_tensor(biases, torch.float32)
self.logits_slice = (self._device_tensor(reqs, torch.int32),
self._device_tensor(tok_ids, torch.int32))
self.logits_slice = (
self._device_tensor(reqs, torch.int32),
self._device_tensor(tok_ids, torch.int32),
)
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
return (torch.tensor(data,
device="cpu",
dtype=dtype,
pin_memory=self.pin_memory).to(device=self.device,
non_blocking=True))
return torch.tensor(
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
).to(device=self.device, non_blocking=True)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if self.biases:
@@ -165,20 +164,19 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
class MinTokensLogitsProcessor(LogitsProcessor):
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
is_pin_memory: bool):
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
):
# index -> (min_toks, output_token_ids, stop_token_ids)
self.device = device
self.pin_memory = is_pin_memory
self.min_toks: dict[int, tuple[int, Sequence[int], set[int]]] = {}
# (req_idx_tensor,eos_tok_id_tensor)
self.logits_slice: tuple[torch.Tensor,
torch.Tensor] = (self._device_tensor(
[], torch.int32),
self._device_tensor(
[], torch.int32))
self.logits_slice: tuple[torch.Tensor, torch.Tensor] = (
self._device_tensor([], torch.int32),
self._device_tensor([], torch.int32),
)
def is_argmax_invariant(self) -> bool:
"""By censoring stop tokens, min-tokens can change the outcome
@@ -187,8 +185,7 @@ class MinTokensLogitsProcessor(LogitsProcessor):
@staticmethod
def add_request(
params: SamplingParams, _: Optional[list[int]],
output_tok_ids: list[int]
params: SamplingParams, _: Optional[list[int]], output_tok_ids: list[int]
) -> Optional[tuple[int, Sequence[int], set[int]]]:
min_tokens = params.min_tokens
if not min_tokens or len(output_tok_ids) >= min_tokens:
@@ -196,13 +193,16 @@ class MinTokensLogitsProcessor(LogitsProcessor):
return min_tokens, output_tok_ids, params.all_stop_token_ids
def update_state(self, batch_update: Optional[BatchUpdate]):
needs_update = process_dict_updates(self.min_toks, batch_update,
self.add_request)
needs_update = process_dict_updates(
self.min_toks, batch_update, self.add_request
)
if self.min_toks:
# Check for any requests that have attained their min tokens.
to_remove = tuple(index for index, (min_toks, out_tok_ids,
_) in self.min_toks.items()
if len(out_tok_ids) >= min_toks)
to_remove = tuple(
index
for index, (min_toks, out_tok_ids, _) in self.min_toks.items()
if len(out_tok_ids) >= min_toks
)
if to_remove:
needs_update = True
for index in to_remove:
@@ -216,15 +216,15 @@ class MinTokensLogitsProcessor(LogitsProcessor):
reqs.extend([req] * len(stop_tok_ids))
tok_ids.extend(stop_tok_ids)
self.logits_slice = (self._device_tensor(reqs, torch.int32),
self._device_tensor(tok_ids, torch.int32))
self.logits_slice = (
self._device_tensor(reqs, torch.int32),
self._device_tensor(tok_ids, torch.int32),
)
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
return (torch.tensor(data,
device="cpu",
dtype=dtype,
pin_memory=self.pin_memory).to(device=self.device,
non_blocking=True))
return torch.tensor(
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
).to(device=self.device, non_blocking=True)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if self.min_toks:
@@ -234,9 +234,9 @@ class MinTokensLogitsProcessor(LogitsProcessor):
def process_dict_updates(
req_entries: dict[int, T], batch_update: Optional[BatchUpdate],
new_state: Callable[[SamplingParams, Optional[list[int]], list[int]],
Optional[T]]
req_entries: dict[int, T],
batch_update: Optional[BatchUpdate],
new_state: Callable[[SamplingParams, Optional[list[int]], list[int]], Optional[T]],
) -> bool:
"""Utility function to update dict state for sparse LogitsProcessors."""
@@ -246,8 +246,7 @@ def process_dict_updates(
updated = False
for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
if (state := new_state(params, prompt_tok_ids,
output_tok_ids)) is not None:
if (state := new_state(params, prompt_tok_ids, output_tok_ids)) is not None:
req_entries[index] = state
updated = True
elif req_entries.pop(index, None) is not None:
+4 -3
View File
@@ -36,6 +36,7 @@ MovedRequest = tuple[int, int, MoveDirectionality]
@dataclass(frozen=True)
class BatchUpdate:
"""Persistent batch state change info for logitsprocs"""
batch_size: int # Current num reqs in batch
# Metadata for requests added to, removed from, and moved
@@ -57,10 +58,10 @@ class BatchUpdate:
class LogitsProcessor(ABC):
@abstractmethod
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
is_pin_memory: bool) -> None:
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
) -> None:
raise NotImplementedError
@abstractmethod
+17 -11
View File
@@ -4,10 +4,12 @@ from collections.abc import Iterator
from itertools import chain
from typing import TYPE_CHECKING, Optional
from vllm.v1.sample.logits_processor.interface import (AddedRequest,
BatchUpdate,
MovedRequest,
RemovedRequest)
from vllm.v1.sample.logits_processor.interface import (
AddedRequest,
BatchUpdate,
MovedRequest,
RemovedRequest,
)
if TYPE_CHECKING:
from vllm.v1.sample.logits_processor.interface import LogitsProcessor
@@ -81,8 +83,9 @@ class BatchUpdateBuilder:
index: request index
"""
if self._is_removed_sorted:
raise RuntimeError("Cannot register new removed request after"
" self.removed has been read.")
raise RuntimeError(
"Cannot register new removed request after self.removed has been read."
)
self._removed.append(index)
self.batch_changed = True
@@ -116,7 +119,7 @@ class BatchUpdateBuilder:
def get_and_reset(self, batch_size: int) -> Optional[BatchUpdate]:
"""Generate a logitsprocs batch update data structure and reset
internal batch update builder state.
Args:
batch_size: current persistent batch size
@@ -146,14 +149,17 @@ class LogitsProcessors:
"""Encapsulates initialized logitsproc objects."""
def __init__(
self,
logitsprocs: Optional[Iterator["LogitsProcessor"]] = None) -> None:
self, logitsprocs: Optional[Iterator["LogitsProcessor"]] = None
) -> None:
self.argmax_invariant: list[LogitsProcessor] = []
self.non_argmax_invariant: list[LogitsProcessor] = []
if logitsprocs:
for logitproc in logitsprocs:
(self.argmax_invariant if logitproc.is_argmax_invariant() else
self.non_argmax_invariant).append(logitproc)
(
self.argmax_invariant
if logitproc.is_argmax_invariant()
else self.non_argmax_invariant
).append(logitproc)
@property
def all(self) -> Iterator["LogitsProcessor"]: