Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -13,15 +13,18 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logits_process import LogitsProcessor as RequestLogitsProcessor
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.logits_processor.builtin import (LogitBiasLogitsProcessor,
|
||||
MinPLogitsProcessor,
|
||||
MinTokensLogitsProcessor,
|
||||
process_dict_updates)
|
||||
from vllm.v1.sample.logits_processor.interface import (BatchUpdate,
|
||||
LogitsProcessor,
|
||||
MoveDirectionality)
|
||||
from vllm.v1.sample.logits_processor.state import (BatchUpdateBuilder,
|
||||
LogitsProcessors)
|
||||
from vllm.v1.sample.logits_processor.builtin import (
|
||||
LogitBiasLogitsProcessor,
|
||||
MinPLogitsProcessor,
|
||||
MinTokensLogitsProcessor,
|
||||
process_dict_updates,
|
||||
)
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
BatchUpdate,
|
||||
LogitsProcessor,
|
||||
MoveDirectionality,
|
||||
)
|
||||
from vllm.v1.sample.logits_processor.state import BatchUpdateBuilder, LogitsProcessors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -30,10 +33,11 @@ logger = init_logger(__name__)
|
||||
|
||||
# Error message when the user tries to initialize vLLM with a pooling model
|
||||
# and custom logitsproces
|
||||
STR_POOLING_REJECTS_LOGITSPROCS = ("Pooling models do not support custom"
|
||||
" logits processors.")
|
||||
STR_POOLING_REJECTS_LOGITSPROCS = (
|
||||
"Pooling models do not support custom logits processors."
|
||||
)
|
||||
|
||||
LOGITSPROCS_GROUP = 'vllm.logits_processors'
|
||||
LOGITSPROCS_GROUP = "vllm.logits_processors"
|
||||
|
||||
BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [
|
||||
MinTokensLogitsProcessor,
|
||||
@@ -54,27 +58,29 @@ def _load_logitsprocs_plugins() -> list[type[LogitsProcessor]]:
|
||||
|
||||
installed_logitsprocs_plugins = entry_points(group=LOGITSPROCS_GROUP)
|
||||
if len(installed_logitsprocs_plugins) == 0:
|
||||
logger.debug("No logitsprocs plugins installed (group %s).",
|
||||
LOGITSPROCS_GROUP)
|
||||
logger.debug("No logitsprocs plugins installed (group %s).", LOGITSPROCS_GROUP)
|
||||
return []
|
||||
|
||||
# Load logitsprocs plugins
|
||||
logger.debug("Loading installed logitsprocs plugins (group %s):",
|
||||
LOGITSPROCS_GROUP)
|
||||
logger.debug("Loading installed logitsprocs plugins (group %s):", LOGITSPROCS_GROUP)
|
||||
classes: list[type[LogitsProcessor]] = []
|
||||
for entrypoint in installed_logitsprocs_plugins:
|
||||
try:
|
||||
logger.debug("- Loading logitproc plugin entrypoint=%s target=%s",
|
||||
entrypoint.name, entrypoint.value)
|
||||
logger.debug(
|
||||
"- Loading logitproc plugin entrypoint=%s target=%s",
|
||||
entrypoint.name,
|
||||
entrypoint.value,
|
||||
)
|
||||
classes.append(entrypoint.load())
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"Failed to load LogitsProcessor plugin {entrypoint}") from e
|
||||
f"Failed to load LogitsProcessor plugin {entrypoint}"
|
||||
) from e
|
||||
return classes
|
||||
|
||||
|
||||
def _load_logitsprocs_by_fqcns(
|
||||
logits_processors: Optional[Sequence[Union[str, type[LogitsProcessor]]]]
|
||||
logits_processors: Optional[Sequence[Union[str, type[LogitsProcessor]]]],
|
||||
) -> list[type[LogitsProcessor]]:
|
||||
"""Load logit processor types, identifying them by fully-qualified class
|
||||
names (FQCNs).
|
||||
@@ -99,13 +105,14 @@ def _load_logitsprocs_by_fqcns(
|
||||
|
||||
logger.debug(
|
||||
"%s additional custom logits processors specified, checking whether "
|
||||
"they need to be loaded.", len(logits_processors))
|
||||
"they need to be loaded.",
|
||||
len(logits_processors),
|
||||
)
|
||||
|
||||
classes: list[type[LogitsProcessor]] = []
|
||||
for ldx, logitproc in enumerate(logits_processors):
|
||||
if isinstance(logitproc, type):
|
||||
logger.debug(" - Already-loaded logit processor: %s",
|
||||
logitproc.__name__)
|
||||
logger.debug(" - Already-loaded logit processor: %s", logitproc.__name__)
|
||||
if not issubclass(logitproc, LogitsProcessor):
|
||||
raise ValueError(
|
||||
f"{logitproc.__name__} is not a subclass of LogitsProcessor"
|
||||
@@ -131,8 +138,7 @@ def _load_logitsprocs_by_fqcns(
|
||||
if not isinstance(obj, type):
|
||||
raise ValueError("Loaded logit processor must be a type.")
|
||||
if not issubclass(obj, LogitsProcessor):
|
||||
raise ValueError(
|
||||
f"{obj.__name__} must be a subclass of LogitsProcessor")
|
||||
raise ValueError(f"{obj.__name__} must be a subclass of LogitsProcessor")
|
||||
classes.append(obj)
|
||||
|
||||
return classes
|
||||
@@ -155,13 +161,13 @@ def _load_custom_logitsprocs(
|
||||
A list of all loaded logitproc types
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_tpu():
|
||||
# No logitsprocs specified by caller
|
||||
# TODO(andy) - vLLM V1 on TPU does not support custom logitsprocs
|
||||
return []
|
||||
|
||||
return (_load_logitsprocs_plugins() +
|
||||
_load_logitsprocs_by_fqcns(logits_processors))
|
||||
return _load_logitsprocs_plugins() + _load_logitsprocs_by_fqcns(logits_processors)
|
||||
|
||||
|
||||
def build_logitsprocs(
|
||||
@@ -174,23 +180,28 @@ def build_logitsprocs(
|
||||
if is_pooling_model:
|
||||
if custom_logitsprocs:
|
||||
raise ValueError(STR_POOLING_REJECTS_LOGITSPROCS)
|
||||
logger.debug("Skipping logits processor loading because pooling models"
|
||||
" do not support logits processors.")
|
||||
logger.debug(
|
||||
"Skipping logits processor loading because pooling models"
|
||||
" do not support logits processors."
|
||||
)
|
||||
return LogitsProcessors()
|
||||
custom_logitsprocs_classes = _load_custom_logitsprocs(custom_logitsprocs)
|
||||
return LogitsProcessors(
|
||||
ctor(vllm_config, device, is_pin_memory) for ctor in itertools.chain(
|
||||
BUILTIN_LOGITS_PROCESSORS, custom_logitsprocs_classes))
|
||||
ctor(vllm_config, device, is_pin_memory)
|
||||
for ctor in itertools.chain(
|
||||
BUILTIN_LOGITS_PROCESSORS, custom_logitsprocs_classes
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class AdapterLogitsProcessor(LogitsProcessor):
|
||||
"""Wrapper for per-request logits processors
|
||||
|
||||
|
||||
To wrap a specific per-request logits processor,
|
||||
* Subclass `AdapterLogitsProcessor`
|
||||
* Implement `self.is_argmax_invariant()` base-class method
|
||||
* Implement `self.new_req_logits_processor(params)`
|
||||
|
||||
|
||||
`self.__init__(vllm_config, device, is_pin_memory)` does not need to be
|
||||
overridden in general. However, to implement custom constructor behavior -
|
||||
especially any logic which operates on or stores `vllm_config`, `device`,
|
||||
@@ -199,8 +210,9 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
`super().__init__(vllm_config, device, is_pin_memory)`
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
|
||||
is_pin_memory: bool):
|
||||
def __init__(
|
||||
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
|
||||
):
|
||||
"""Subclass must invoke
|
||||
`super().__init__(vllm_config, device, is_pin_memory)`.
|
||||
|
||||
@@ -236,7 +248,7 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
Returns:
|
||||
None if logits processor should not be applied to request; otherwise
|
||||
returns a `RequestLogitsProcessor` instance
|
||||
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -257,11 +269,14 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
|
||||
Returns:
|
||||
logits processor partial[Tensor] or None
|
||||
|
||||
|
||||
"""
|
||||
if req_lp := self.new_req_logits_processor(params):
|
||||
args = [prompt_ids, output_ids] if (len(
|
||||
inspect.signature(req_lp).parameters) == 3) else [output_ids]
|
||||
args = (
|
||||
[prompt_ids, output_ids]
|
||||
if (len(inspect.signature(req_lp).parameters) == 3)
|
||||
else [output_ids]
|
||||
)
|
||||
return partial(req_lp, *args)
|
||||
return None
|
||||
|
||||
@@ -286,9 +301,16 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LogitsProcessor", "LogitBiasLogitsProcessor", "MinPLogitsProcessor",
|
||||
"MinTokensLogitsProcessor", "BatchUpdate", "BatchUpdateBuilder",
|
||||
"MoveDirectionality", "LogitsProcessors", "build_logitsprocs",
|
||||
"STR_POOLING_REJECTS_LOGITSPROCS", "LOGITSPROCS_GROUP",
|
||||
"AdapterLogitsProcessor"
|
||||
"LogitsProcessor",
|
||||
"LogitBiasLogitsProcessor",
|
||||
"MinPLogitsProcessor",
|
||||
"MinTokensLogitsProcessor",
|
||||
"BatchUpdate",
|
||||
"BatchUpdateBuilder",
|
||||
"MoveDirectionality",
|
||||
"LogitsProcessors",
|
||||
"build_logitsprocs",
|
||||
"STR_POOLING_REJECTS_LOGITSPROCS",
|
||||
"LOGITSPROCS_GROUP",
|
||||
"AdapterLogitsProcessor",
|
||||
]
|
||||
|
||||
@@ -6,9 +6,11 @@ from typing import TYPE_CHECKING, Callable, Optional, TypeVar
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.sample.logits_processor.interface import (BatchUpdate,
|
||||
LogitsProcessor,
|
||||
MoveDirectionality)
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
BatchUpdate,
|
||||
LogitsProcessor,
|
||||
MoveDirectionality,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -17,25 +19,24 @@ T = TypeVar("T")
|
||||
|
||||
|
||||
class MinPLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
|
||||
is_pin_memory: bool):
|
||||
def __init__(
|
||||
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
|
||||
):
|
||||
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
self.min_p_count: int = 0
|
||||
|
||||
self.min_p_cpu_tensor = torch.zeros((max_num_reqs, ),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
pin_memory=is_pin_memory)
|
||||
self.min_p_cpu_tensor = torch.zeros(
|
||||
(max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=is_pin_memory
|
||||
)
|
||||
self.min_p_cpu = self.min_p_cpu_tensor.numpy()
|
||||
|
||||
self.use_double_tensor = torch.device(device).type != "cpu"
|
||||
|
||||
if self.use_double_tensor:
|
||||
# Pre-allocated device tensor
|
||||
self.min_p_device: torch.Tensor = torch.empty((max_num_reqs, ),
|
||||
dtype=torch.float32,
|
||||
device=device)
|
||||
self.min_p_device: torch.Tensor = torch.empty(
|
||||
(max_num_reqs,), dtype=torch.float32, device=device
|
||||
)
|
||||
else:
|
||||
self.min_p_device = self.min_p_cpu_tensor
|
||||
# Current slice of the device tensor
|
||||
@@ -93,8 +94,7 @@ class MinPLogitsProcessor(LogitsProcessor):
|
||||
if self.min_p_count and (needs_update or self.min_p.shape[0] != size):
|
||||
self.min_p = self.min_p_device[:size]
|
||||
if self.use_double_tensor:
|
||||
self.min_p.copy_(self.min_p_cpu_tensor[:size],
|
||||
non_blocking=True)
|
||||
self.min_p.copy_(self.min_p_cpu_tensor[:size], non_blocking=True)
|
||||
self.min_p.unsqueeze_(1)
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
@@ -104,28 +104,27 @@ class MinPLogitsProcessor(LogitsProcessor):
|
||||
# Convert logits to probability distribution
|
||||
probability_values = torch.nn.functional.softmax(logits, dim=-1)
|
||||
# Calculate maximum probabilities per sequence
|
||||
max_probabilities = torch.amax(probability_values,
|
||||
dim=-1,
|
||||
keepdim=True)
|
||||
max_probabilities = torch.amax(probability_values, dim=-1, keepdim=True)
|
||||
# Adjust min_p
|
||||
adjusted_min_p = max_probabilities.mul_(self.min_p)
|
||||
# Identify valid tokens using threshold comparison
|
||||
invalid_token_mask = probability_values < adjusted_min_p
|
||||
# Apply mask using boolean indexing
|
||||
logits[invalid_token_mask] = -float('inf')
|
||||
logits[invalid_token_mask] = -float("inf")
|
||||
return logits
|
||||
|
||||
|
||||
class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def __init__(self, _, device: torch.device, is_pin_memory: bool):
|
||||
self.device = device
|
||||
self.pin_memory = is_pin_memory
|
||||
self.biases: dict[int, dict[int, float]] = {}
|
||||
|
||||
self.bias_tensor: torch.Tensor = torch.tensor(())
|
||||
self.logits_slice = (self._device_tensor([], torch.int32),
|
||||
self._device_tensor([], torch.int32))
|
||||
self.logits_slice = (
|
||||
self._device_tensor([], torch.int32),
|
||||
self._device_tensor([], torch.int32),
|
||||
)
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
"""Logit bias can rebalance token probabilities and change the
|
||||
@@ -134,8 +133,8 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
needs_update = process_dict_updates(
|
||||
self.biases, batch_update,
|
||||
lambda params, _, __: params.logit_bias or None)
|
||||
self.biases, batch_update, lambda params, _, __: params.logit_bias or None
|
||||
)
|
||||
|
||||
# Update tensors if needed.
|
||||
if needs_update:
|
||||
@@ -148,15 +147,15 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
biases.extend(lb.values())
|
||||
|
||||
self.bias_tensor = self._device_tensor(biases, torch.float32)
|
||||
self.logits_slice = (self._device_tensor(reqs, torch.int32),
|
||||
self._device_tensor(tok_ids, torch.int32))
|
||||
self.logits_slice = (
|
||||
self._device_tensor(reqs, torch.int32),
|
||||
self._device_tensor(tok_ids, torch.int32),
|
||||
)
|
||||
|
||||
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
|
||||
return (torch.tensor(data,
|
||||
device="cpu",
|
||||
dtype=dtype,
|
||||
pin_memory=self.pin_memory).to(device=self.device,
|
||||
non_blocking=True))
|
||||
return torch.tensor(
|
||||
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
|
||||
).to(device=self.device, non_blocking=True)
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if self.biases:
|
||||
@@ -165,20 +164,19 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
|
||||
|
||||
class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
|
||||
is_pin_memory: bool):
|
||||
def __init__(
|
||||
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
|
||||
):
|
||||
# index -> (min_toks, output_token_ids, stop_token_ids)
|
||||
self.device = device
|
||||
self.pin_memory = is_pin_memory
|
||||
self.min_toks: dict[int, tuple[int, Sequence[int], set[int]]] = {}
|
||||
|
||||
# (req_idx_tensor,eos_tok_id_tensor)
|
||||
self.logits_slice: tuple[torch.Tensor,
|
||||
torch.Tensor] = (self._device_tensor(
|
||||
[], torch.int32),
|
||||
self._device_tensor(
|
||||
[], torch.int32))
|
||||
self.logits_slice: tuple[torch.Tensor, torch.Tensor] = (
|
||||
self._device_tensor([], torch.int32),
|
||||
self._device_tensor([], torch.int32),
|
||||
)
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
"""By censoring stop tokens, min-tokens can change the outcome
|
||||
@@ -187,8 +185,7 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
@staticmethod
|
||||
def add_request(
|
||||
params: SamplingParams, _: Optional[list[int]],
|
||||
output_tok_ids: list[int]
|
||||
params: SamplingParams, _: Optional[list[int]], output_tok_ids: list[int]
|
||||
) -> Optional[tuple[int, Sequence[int], set[int]]]:
|
||||
min_tokens = params.min_tokens
|
||||
if not min_tokens or len(output_tok_ids) >= min_tokens:
|
||||
@@ -196,13 +193,16 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
return min_tokens, output_tok_ids, params.all_stop_token_ids
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
needs_update = process_dict_updates(self.min_toks, batch_update,
|
||||
self.add_request)
|
||||
needs_update = process_dict_updates(
|
||||
self.min_toks, batch_update, self.add_request
|
||||
)
|
||||
if self.min_toks:
|
||||
# Check for any requests that have attained their min tokens.
|
||||
to_remove = tuple(index for index, (min_toks, out_tok_ids,
|
||||
_) in self.min_toks.items()
|
||||
if len(out_tok_ids) >= min_toks)
|
||||
to_remove = tuple(
|
||||
index
|
||||
for index, (min_toks, out_tok_ids, _) in self.min_toks.items()
|
||||
if len(out_tok_ids) >= min_toks
|
||||
)
|
||||
if to_remove:
|
||||
needs_update = True
|
||||
for index in to_remove:
|
||||
@@ -216,15 +216,15 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
reqs.extend([req] * len(stop_tok_ids))
|
||||
tok_ids.extend(stop_tok_ids)
|
||||
|
||||
self.logits_slice = (self._device_tensor(reqs, torch.int32),
|
||||
self._device_tensor(tok_ids, torch.int32))
|
||||
self.logits_slice = (
|
||||
self._device_tensor(reqs, torch.int32),
|
||||
self._device_tensor(tok_ids, torch.int32),
|
||||
)
|
||||
|
||||
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
|
||||
return (torch.tensor(data,
|
||||
device="cpu",
|
||||
dtype=dtype,
|
||||
pin_memory=self.pin_memory).to(device=self.device,
|
||||
non_blocking=True))
|
||||
return torch.tensor(
|
||||
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
|
||||
).to(device=self.device, non_blocking=True)
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if self.min_toks:
|
||||
@@ -234,9 +234,9 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
|
||||
def process_dict_updates(
|
||||
req_entries: dict[int, T], batch_update: Optional[BatchUpdate],
|
||||
new_state: Callable[[SamplingParams, Optional[list[int]], list[int]],
|
||||
Optional[T]]
|
||||
req_entries: dict[int, T],
|
||||
batch_update: Optional[BatchUpdate],
|
||||
new_state: Callable[[SamplingParams, Optional[list[int]], list[int]], Optional[T]],
|
||||
) -> bool:
|
||||
"""Utility function to update dict state for sparse LogitsProcessors."""
|
||||
|
||||
@@ -246,8 +246,7 @@ def process_dict_updates(
|
||||
|
||||
updated = False
|
||||
for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
|
||||
if (state := new_state(params, prompt_tok_ids,
|
||||
output_tok_ids)) is not None:
|
||||
if (state := new_state(params, prompt_tok_ids, output_tok_ids)) is not None:
|
||||
req_entries[index] = state
|
||||
updated = True
|
||||
elif req_entries.pop(index, None) is not None:
|
||||
|
||||
@@ -36,6 +36,7 @@ MovedRequest = tuple[int, int, MoveDirectionality]
|
||||
@dataclass(frozen=True)
|
||||
class BatchUpdate:
|
||||
"""Persistent batch state change info for logitsprocs"""
|
||||
|
||||
batch_size: int # Current num reqs in batch
|
||||
|
||||
# Metadata for requests added to, removed from, and moved
|
||||
@@ -57,10 +58,10 @@ class BatchUpdate:
|
||||
|
||||
|
||||
class LogitsProcessor(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
|
||||
is_pin_memory: bool) -> None:
|
||||
def __init__(
|
||||
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -4,10 +4,12 @@ from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from vllm.v1.sample.logits_processor.interface import (AddedRequest,
|
||||
BatchUpdate,
|
||||
MovedRequest,
|
||||
RemovedRequest)
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
AddedRequest,
|
||||
BatchUpdate,
|
||||
MovedRequest,
|
||||
RemovedRequest,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.sample.logits_processor.interface import LogitsProcessor
|
||||
@@ -81,8 +83,9 @@ class BatchUpdateBuilder:
|
||||
index: request index
|
||||
"""
|
||||
if self._is_removed_sorted:
|
||||
raise RuntimeError("Cannot register new removed request after"
|
||||
" self.removed has been read.")
|
||||
raise RuntimeError(
|
||||
"Cannot register new removed request after self.removed has been read."
|
||||
)
|
||||
self._removed.append(index)
|
||||
self.batch_changed = True
|
||||
|
||||
@@ -116,7 +119,7 @@ class BatchUpdateBuilder:
|
||||
def get_and_reset(self, batch_size: int) -> Optional[BatchUpdate]:
|
||||
"""Generate a logitsprocs batch update data structure and reset
|
||||
internal batch update builder state.
|
||||
|
||||
|
||||
Args:
|
||||
batch_size: current persistent batch size
|
||||
|
||||
@@ -146,14 +149,17 @@ class LogitsProcessors:
|
||||
"""Encapsulates initialized logitsproc objects."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
logitsprocs: Optional[Iterator["LogitsProcessor"]] = None) -> None:
|
||||
self, logitsprocs: Optional[Iterator["LogitsProcessor"]] = None
|
||||
) -> None:
|
||||
self.argmax_invariant: list[LogitsProcessor] = []
|
||||
self.non_argmax_invariant: list[LogitsProcessor] = []
|
||||
if logitsprocs:
|
||||
for logitproc in logitsprocs:
|
||||
(self.argmax_invariant if logitproc.is_argmax_invariant() else
|
||||
self.non_argmax_invariant).append(logitproc)
|
||||
(
|
||||
self.argmax_invariant
|
||||
if logitproc.is_argmax_invariant()
|
||||
else self.non_argmax_invariant
|
||||
).append(logitproc)
|
||||
|
||||
@property
|
||||
def all(self) -> Iterator["LogitsProcessor"]:
|
||||
|
||||
@@ -11,7 +11,6 @@ from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
|
||||
@dataclass
|
||||
class SamplingMetadata:
|
||||
|
||||
temperature: Optional[torch.Tensor]
|
||||
all_greedy: bool
|
||||
all_random: bool
|
||||
|
||||
@@ -35,5 +35,4 @@ def apply_bad_words(
|
||||
past_tokens_ids: list[list[int]],
|
||||
) -> None:
|
||||
for i, bad_words_ids in bad_words_token_ids.items():
|
||||
_apply_bad_words_single_batch(logits[i], bad_words_ids,
|
||||
past_tokens_ids[i])
|
||||
_apply_bad_words_single_batch(logits[i], bad_words_ids, past_tokens_ids[i])
|
||||
|
||||
@@ -8,8 +8,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def batched_count_greater_than(x: torch.Tensor,
|
||||
values: torch.Tensor) -> torch.Tensor:
|
||||
def batched_count_greater_than(x: torch.Tensor, values: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Counts elements in each row of x that are greater than the corresponding
|
||||
value in values. Use torch.compile to generate an optimized kernel for
|
||||
|
||||
@@ -19,15 +19,20 @@ def apply_all_penalties(
|
||||
Applies presence, frequency and repetition penalties to the logits.
|
||||
"""
|
||||
_, vocab_size = logits.shape
|
||||
output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
|
||||
logits.device)
|
||||
return apply_penalties(logits, prompt_token_ids, output_tokens_t,
|
||||
presence_penalties, frequency_penalties,
|
||||
repetition_penalties)
|
||||
output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size, logits.device)
|
||||
return apply_penalties(
|
||||
logits,
|
||||
prompt_token_ids,
|
||||
output_tokens_t,
|
||||
presence_penalties,
|
||||
frequency_penalties,
|
||||
repetition_penalties,
|
||||
)
|
||||
|
||||
|
||||
def _convert_to_tensors(output_token_ids: list[list[int]], vocab_size: int,
|
||||
device: torch.device) -> torch.Tensor:
|
||||
def _convert_to_tensors(
|
||||
output_token_ids: list[list[int]], vocab_size: int, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Convert the different list data structures to tensors.
|
||||
"""
|
||||
|
||||
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
import flashinfer.sampling
|
||||
|
||||
is_flashinfer_available = True
|
||||
except ImportError:
|
||||
is_flashinfer_available = False
|
||||
@@ -34,14 +35,17 @@ class TopKTopPSampler(nn.Module):
|
||||
self.logprobs_mode = logprobs_mode
|
||||
# flashinfer optimization does not apply if intermediate
|
||||
# logprobs/logits after top_k/top_p need to be returned
|
||||
if logprobs_mode not in ("processed_logits", "processed_logprobs"
|
||||
) and current_platform.is_cuda():
|
||||
if (
|
||||
logprobs_mode not in ("processed_logits", "processed_logprobs")
|
||||
and current_platform.is_cuda()
|
||||
):
|
||||
if is_flashinfer_available:
|
||||
flashinfer_version = flashinfer.__version__
|
||||
if version.parse(flashinfer_version) < version.parse("0.2.3"):
|
||||
logger.warning_once(
|
||||
"FlashInfer version >= 0.2.3 required. "
|
||||
"Falling back to default sampling implementation.")
|
||||
"Falling back to default sampling implementation."
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
elif envs.VLLM_USE_FLASHINFER_SAMPLER is not False:
|
||||
# NOTE(woosuk): The V0 sampler doesn't use FlashInfer for
|
||||
@@ -52,21 +56,22 @@ class TopKTopPSampler(nn.Module):
|
||||
# None means False, while in V1, None means True. This is
|
||||
# why we use the condition
|
||||
# `envs.VLLM_USE_FLASHINFER_SAMPLER is not False` here.
|
||||
logger.info_once(
|
||||
"Using FlashInfer for top-p & top-k sampling.")
|
||||
logger.info_once("Using FlashInfer for top-p & top-k sampling.")
|
||||
self.forward = self.forward_cuda
|
||||
else:
|
||||
logger.warning_once(
|
||||
"FlashInfer is available, but it is not enabled. "
|
||||
"Falling back to the PyTorch-native implementation of "
|
||||
"top-p & top-k sampling. For the best performance, "
|
||||
"please set VLLM_USE_FLASHINFER_SAMPLER=1.")
|
||||
"please set VLLM_USE_FLASHINFER_SAMPLER=1."
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
else:
|
||||
logger.warning_once(
|
||||
"FlashInfer is not available. Falling back to the PyTorch-"
|
||||
"native implementation of top-p & top-k sampling. For the "
|
||||
"best performance, please install FlashInfer.")
|
||||
"best performance, please install FlashInfer."
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
elif current_platform.is_cpu():
|
||||
self.forward = self.forward_cpu
|
||||
@@ -109,13 +114,15 @@ class TopKTopPSampler(nn.Module):
|
||||
# CPU-GPU synchronization while `flashinfer_sample` does.
|
||||
if (k is None and p is None) or generators:
|
||||
if generators:
|
||||
logger.debug_once("FlashInfer 0.2.3+ does not support "
|
||||
"per-request generators. Falling back to "
|
||||
"PyTorch-native implementation.")
|
||||
logger.debug_once(
|
||||
"FlashInfer 0.2.3+ does not support "
|
||||
"per-request generators. Falling back to "
|
||||
"PyTorch-native implementation."
|
||||
)
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
assert self.logprobs_mode not in (
|
||||
"processed_logits", "processed_logprobs"
|
||||
), "FlashInfer does not support returning logits/logprobs"
|
||||
assert self.logprobs_mode not in ("processed_logits", "processed_logprobs"), (
|
||||
"FlashInfer does not support returning logits/logprobs"
|
||||
)
|
||||
# flashinfer sampling functions expect contiguous logits.
|
||||
# In flex_attn/triton_attn fp32 inference, logits can be non-contiguous
|
||||
# because of slicing operation in logits_processor.
|
||||
@@ -278,15 +285,18 @@ def flashinfer_sample(
|
||||
# Top-p only.
|
||||
probs = logits.softmax(dim=-1, dtype=torch.float32)
|
||||
next_token_ids = flashinfer.sampling.top_p_sampling_from_probs(
|
||||
probs, p, deterministic=True)
|
||||
probs, p, deterministic=True
|
||||
)
|
||||
elif p is None:
|
||||
# Top-k only.
|
||||
probs = logits.softmax(dim=-1, dtype=torch.float32)
|
||||
next_token_ids = flashinfer.sampling.top_k_sampling_from_probs(
|
||||
probs, k, deterministic=True)
|
||||
probs, k, deterministic=True
|
||||
)
|
||||
else:
|
||||
# Both top-k and top-p.
|
||||
next_token_ids = flashinfer.sampling.top_k_top_p_sampling_from_logits(
|
||||
logits, k, p, deterministic=True)
|
||||
logits, k, p, deterministic=True
|
||||
)
|
||||
|
||||
return next_token_ids.view(-1)
|
||||
|
||||
@@ -54,7 +54,7 @@ class RejectionSampler(nn.Module):
|
||||
bonus_token_ids: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> torch.Tensor:
|
||||
'''
|
||||
"""
|
||||
Args:
|
||||
metadata:
|
||||
Metadata for spec decoding.
|
||||
@@ -81,7 +81,7 @@ class RejectionSampler(nn.Module):
|
||||
Returns:
|
||||
output_token_ids (torch.Tensor):
|
||||
A tensor containing the final output token IDs.
|
||||
'''
|
||||
"""
|
||||
assert metadata.max_spec_len <= MAX_SPEC_LEN
|
||||
# [num_tokens, vocab_size]
|
||||
# NOTE(woosuk): `target_logits` can be updated in place inside the
|
||||
@@ -123,11 +123,11 @@ class RejectionSampler(nn.Module):
|
||||
"""
|
||||
output_token_ids_np = output_token_ids.cpu().numpy()
|
||||
# Create mask for valid tokens.
|
||||
valid_mask = ((output_token_ids_np != PLACEHOLDER_TOKEN_ID) &
|
||||
(output_token_ids_np < vocab_size))
|
||||
valid_mask = (output_token_ids_np != PLACEHOLDER_TOKEN_ID) & (
|
||||
output_token_ids_np < vocab_size
|
||||
)
|
||||
outputs = [
|
||||
row[valid_mask[i]].tolist()
|
||||
for i, row in enumerate(output_token_ids_np)
|
||||
row[valid_mask[i]].tolist() for i, row in enumerate(output_token_ids_np)
|
||||
]
|
||||
return outputs
|
||||
|
||||
@@ -178,7 +178,7 @@ def rejection_sample(
|
||||
if not sampling_metadata.all_random:
|
||||
# Rejection sampling for greedy sampling requests.
|
||||
target_argmax = target_probs.argmax(dim=-1)
|
||||
rejection_greedy_sample_kernel[(batch_size, )](
|
||||
rejection_greedy_sample_kernel[(batch_size,)](
|
||||
output_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
@@ -213,7 +213,7 @@ def rejection_sample(
|
||||
)
|
||||
|
||||
# Rejection sampling for random sampling requests.
|
||||
rejection_random_sample_kernel[(batch_size, )](
|
||||
rejection_random_sample_kernel[(batch_size,)](
|
||||
output_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
@@ -320,7 +320,7 @@ def expand_batch_to_tokens(
|
||||
batch_size = x.shape[0]
|
||||
assert cu_num_tokens.shape[0] == batch_size
|
||||
expanded_x = x.new_empty(num_tokens)
|
||||
expand_kernel[(batch_size, )](
|
||||
expand_kernel[(batch_size,)](
|
||||
expanded_x,
|
||||
x,
|
||||
cu_num_tokens,
|
||||
@@ -368,7 +368,7 @@ def generate_uniform_probs(
|
||||
# https://github.com/pytorch/pytorch/issues/16706. Using float64
|
||||
# mitigates the issue.
|
||||
uniform_probs = torch.rand(
|
||||
(num_tokens, ),
|
||||
(num_tokens,),
|
||||
dtype=torch.float64,
|
||||
device=device,
|
||||
)
|
||||
@@ -464,8 +464,10 @@ def rejection_greedy_sample_kernel(
|
||||
if not rejected:
|
||||
draft_token_id = tl.load(draft_token_ids_ptr + start_idx + pos)
|
||||
target_argmax_id = tl.load(target_argmax_ptr + start_idx + pos)
|
||||
tl.store(output_token_ids_ptr + req_idx * (max_spec_len + 1) + pos,
|
||||
target_argmax_id)
|
||||
tl.store(
|
||||
output_token_ids_ptr + req_idx * (max_spec_len + 1) + pos,
|
||||
target_argmax_id,
|
||||
)
|
||||
if draft_token_id != target_argmax_id:
|
||||
# Reject.
|
||||
rejected = True
|
||||
@@ -474,8 +476,9 @@ def rejection_greedy_sample_kernel(
|
||||
# If all tokens are accepted, append the bonus token.
|
||||
bonus_token_id = tl.load(bonus_token_ids_ptr + req_idx)
|
||||
tl.store(
|
||||
output_token_ids_ptr + req_idx * (max_spec_len + 1) +
|
||||
num_draft_tokens, bonus_token_id)
|
||||
output_token_ids_ptr + req_idx * (max_spec_len + 1) + num_draft_tokens,
|
||||
bonus_token_id,
|
||||
)
|
||||
|
||||
|
||||
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
|
||||
@@ -514,12 +517,12 @@ def rejection_random_sample_kernel(
|
||||
if NO_DRAFT_PROBS:
|
||||
draft_prob = 1
|
||||
else:
|
||||
draft_prob = tl.load(draft_probs_ptr +
|
||||
(start_idx + pos) * vocab_size +
|
||||
draft_token_id)
|
||||
target_prob = tl.load(target_probs_ptr +
|
||||
(start_idx + pos) * vocab_size +
|
||||
draft_token_id)
|
||||
draft_prob = tl.load(
|
||||
draft_probs_ptr + (start_idx + pos) * vocab_size + draft_token_id
|
||||
)
|
||||
target_prob = tl.load(
|
||||
target_probs_ptr + (start_idx + pos) * vocab_size + draft_token_id
|
||||
)
|
||||
uniform_prob = tl.load(uniform_probs_ptr + start_idx + pos)
|
||||
# NOTE(woosuk): While the draft probability should never be 0,
|
||||
# we check it to avoid NaNs. If it happens to be 0, we reject.
|
||||
@@ -530,15 +533,17 @@ def rejection_random_sample_kernel(
|
||||
# Reject. Use recovered token.
|
||||
rejected = True
|
||||
token_id = tl.load(recovered_token_ids_ptr + start_idx + pos)
|
||||
tl.store(output_token_ids_ptr + req_idx * (max_spec_len + 1) + pos,
|
||||
token_id)
|
||||
tl.store(
|
||||
output_token_ids_ptr + req_idx * (max_spec_len + 1) + pos, token_id
|
||||
)
|
||||
|
||||
if not rejected:
|
||||
# If all tokens are accepted, append the bonus token.
|
||||
bonus_token_id = tl.load(bonus_token_ids_ptr + req_idx)
|
||||
tl.store(
|
||||
output_token_ids_ptr + req_idx * (max_spec_len + 1) +
|
||||
num_draft_tokens, bonus_token_id)
|
||||
output_token_ids_ptr + req_idx * (max_spec_len + 1) + num_draft_tokens,
|
||||
bonus_token_id,
|
||||
)
|
||||
|
||||
|
||||
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
|
||||
@@ -562,9 +567,7 @@ def expand_kernel(
|
||||
src_val = tl.load(input_ptr + req_idx)
|
||||
src_val = tl.where(src_val == replace_from, replace_to, src_val)
|
||||
offset = tl.arange(0, MAX_NUM_TOKENS)
|
||||
tl.store(output_ptr + start_idx + offset,
|
||||
src_val,
|
||||
mask=offset < num_tokens)
|
||||
tl.store(output_ptr + start_idx + offset, src_val, mask=offset < num_tokens)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -595,26 +598,30 @@ def sample_recovered_tokens_kernel(
|
||||
vocab_offset = tl.arange(0, PADDED_VOCAB_SIZE)
|
||||
if NO_DRAFT_PROBS:
|
||||
draft_token_id = tl.load(draft_token_ids_ptr + start_idx + pos)
|
||||
prob = tl.load(target_probs_ptr + (start_idx + pos) * vocab_size +
|
||||
vocab_offset,
|
||||
mask=((vocab_offset < vocab_size) &
|
||||
(vocab_offset != draft_token_id)),
|
||||
other=0)
|
||||
prob = tl.load(
|
||||
target_probs_ptr + (start_idx + pos) * vocab_size + vocab_offset,
|
||||
mask=((vocab_offset < vocab_size) & (vocab_offset != draft_token_id)),
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
draft_prob = tl.load(draft_probs_ptr + (start_idx + pos) * vocab_size +
|
||||
vocab_offset,
|
||||
mask=vocab_offset < vocab_size,
|
||||
other=0)
|
||||
target_prob = tl.load(target_probs_ptr +
|
||||
(start_idx + pos) * vocab_size + vocab_offset,
|
||||
mask=vocab_offset < vocab_size,
|
||||
other=0)
|
||||
draft_prob = tl.load(
|
||||
draft_probs_ptr + (start_idx + pos) * vocab_size + vocab_offset,
|
||||
mask=vocab_offset < vocab_size,
|
||||
other=0,
|
||||
)
|
||||
target_prob = tl.load(
|
||||
target_probs_ptr + (start_idx + pos) * vocab_size + vocab_offset,
|
||||
mask=vocab_offset < vocab_size,
|
||||
other=0,
|
||||
)
|
||||
prob = tl.maximum(target_prob - draft_prob, 0)
|
||||
# NOTE(woosuk): We don't need `prob = prob / tl.sum(prob)` here because
|
||||
# `tl.argmax` will select the maximum value.
|
||||
|
||||
q = tl.load(q_ptr + req_idx * vocab_size + vocab_offset,
|
||||
mask=vocab_offset < vocab_size,
|
||||
other=float("-inf"))
|
||||
q = tl.load(
|
||||
q_ptr + req_idx * vocab_size + vocab_offset,
|
||||
mask=vocab_offset < vocab_size,
|
||||
other=float("-inf"),
|
||||
)
|
||||
recovered_id = tl.argmax(prob / q, axis=-1)
|
||||
tl.store(output_token_ids_ptr + start_idx + pos, recovered_id)
|
||||
|
||||
+32
-32
@@ -24,39 +24,39 @@ class Sampler(nn.Module):
|
||||
A layer that samples the next tokens from the model's outputs
|
||||
with the following steps in order:
|
||||
|
||||
1. If logprobs are requested:
|
||||
1. If logprobs are requested:
|
||||
a) If `logprobs_mode` is `raw_logprobs`, compute logprobs
|
||||
as the final logprobs to return.
|
||||
as the final logprobs to return.
|
||||
b) If `logprobs_mode` is `raw_logits`, clone the logits
|
||||
as the final logprobs to return.
|
||||
2. Convert logits to float32.
|
||||
3. Apply allowed token ids whitelist.
|
||||
4. Apply bad words exclusion.
|
||||
as the final logprobs to return.
|
||||
2. Convert logits to float32.
|
||||
3. Apply allowed token ids whitelist.
|
||||
4. Apply bad words exclusion.
|
||||
5. Apply logit processors which are not argmax-invariant,
|
||||
i.e. that can impact greedy sampling.
|
||||
a) Min tokens processor
|
||||
b) Logit bias processor
|
||||
6. Apply penalties
|
||||
a) Repetition penalty
|
||||
b) Frequency penalty
|
||||
c) Presence penalty
|
||||
7. Sample the next tokens. `sample` method performs the following steps:
|
||||
i.e. that can impact greedy sampling.
|
||||
a) Min tokens processor
|
||||
b) Logit bias processor
|
||||
6. Apply penalties
|
||||
a) Repetition penalty
|
||||
b) Frequency penalty
|
||||
c) Presence penalty
|
||||
7. Sample the next tokens. `sample` method performs the following steps:
|
||||
a) If not `all_random`, perform greedy sampling. If `all_greedy`,
|
||||
return the greedily sampled tokens and final logprobs if requested.
|
||||
b) Apply temperature.
|
||||
return the greedily sampled tokens and final logprobs if requested.
|
||||
b) Apply temperature.
|
||||
c) Apply logit processors which are argmax-invariant, by default
|
||||
the min_p processor.
|
||||
d) Apply top_k and/or top_p.
|
||||
e) Sample the next tokens with the probability distribution.
|
||||
the min_p processor.
|
||||
d) Apply top_k and/or top_p.
|
||||
e) Sample the next tokens with the probability distribution.
|
||||
f) If `all_random` or temperature >= epsilon (1e-5), return the
|
||||
randomly sampled tokens and final logprobs if requested. Else,
|
||||
return the greedily sampled tokens and logprobs if requested.
|
||||
return the greedily sampled tokens and logprobs if requested.
|
||||
8. Gather the logprobs of the top `max_num_logprobs` and sampled token
|
||||
(if requested). Note that if the sampled token is within the top
|
||||
`max_num_logprobs`, the logprob will be eventually merged in
|
||||
`LogprobsProcessor` during output processing. Therefore, the
|
||||
final output may contain either `max_num_logprobs + 1` or
|
||||
`max_num_logprobs` logprobs.
|
||||
`max_num_logprobs` logprobs.
|
||||
9. Return the final `SamplerOutput`.
|
||||
"""
|
||||
|
||||
@@ -108,8 +108,11 @@ class Sampler(nn.Module):
|
||||
|
||||
# Gather the logprobs of the topk and sampled token (if requested).
|
||||
# Get logprobs and rank tensors (if requested)
|
||||
logprobs_tensors = None if num_logprobs is None else \
|
||||
self.gather_logprobs(raw_logprobs, num_logprobs, token_ids=sampled)
|
||||
logprobs_tensors = (
|
||||
None
|
||||
if num_logprobs is None
|
||||
else self.gather_logprobs(raw_logprobs, num_logprobs, token_ids=sampled)
|
||||
)
|
||||
|
||||
# Use int32 to reduce the tensor size.
|
||||
sampled = sampled.to(torch.int32)
|
||||
@@ -150,8 +153,7 @@ class Sampler(nn.Module):
|
||||
may update the logits tensor in-place.
|
||||
"""
|
||||
|
||||
assert not (sampling_metadata.all_greedy
|
||||
and sampling_metadata.all_random)
|
||||
assert not (sampling_metadata.all_greedy and sampling_metadata.all_random)
|
||||
if sampling_metadata.all_random:
|
||||
greedy_sampled = None
|
||||
else:
|
||||
@@ -168,8 +170,9 @@ class Sampler(nn.Module):
|
||||
assert sampling_metadata.temperature is not None
|
||||
|
||||
# Apply temperature.
|
||||
logits = self.apply_temperature(logits, sampling_metadata.temperature,
|
||||
sampling_metadata.all_random)
|
||||
logits = self.apply_temperature(
|
||||
logits, sampling_metadata.temperature, sampling_metadata.all_random
|
||||
)
|
||||
|
||||
# Apply logits processors that only apply to random sampling
|
||||
# (argmax invariant)
|
||||
@@ -224,9 +227,7 @@ class Sampler(nn.Module):
|
||||
"""
|
||||
assert token_ids.dtype == torch.int64
|
||||
# Find the topK values.
|
||||
topk_logprobs, topk_indices = torch.topk(logprobs,
|
||||
num_logprobs,
|
||||
dim=-1)
|
||||
topk_logprobs, topk_indices = torch.topk(logprobs, num_logprobs, dim=-1)
|
||||
|
||||
# Get with the logprob of the prompt or sampled token.
|
||||
token_ids = token_ids.unsqueeze(-1)
|
||||
@@ -267,8 +268,7 @@ class Sampler(nn.Module):
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> torch.Tensor:
|
||||
if sampling_metadata.allowed_token_ids_mask is not None:
|
||||
logits.masked_fill_(sampling_metadata.allowed_token_ids_mask,
|
||||
float("-inf"))
|
||||
logits.masked_fill_(sampling_metadata.allowed_token_ids_mask, float("-inf"))
|
||||
return logits
|
||||
|
||||
def apply_bad_words(
|
||||
|
||||
@@ -48,15 +48,13 @@ class TPUSupportedSamplingMetadata:
|
||||
|
||||
min_tokens = None # impl is not vectorized
|
||||
|
||||
logit_bias: list[Optional[dict[int, float]]] = field(
|
||||
default_factory=lambda: list())
|
||||
logit_bias: list[Optional[dict[int, float]]] = field(default_factory=lambda: list())
|
||||
|
||||
allowed_token_ids_mask = None
|
||||
bad_words_token_ids = None
|
||||
|
||||
# Generator not supported by xla
|
||||
_generators: dict[int,
|
||||
torch.Generator] = field(default_factory=lambda: dict())
|
||||
_generators: dict[int, torch.Generator] = field(default_factory=lambda: dict())
|
||||
|
||||
@property
|
||||
def generators(self) -> dict[int, torch.Generator]:
|
||||
@@ -69,13 +67,13 @@ class TPUSupportedSamplingMetadata:
|
||||
input_batch: InputBatch,
|
||||
padded_num_reqs: int,
|
||||
xla_device: torch.device,
|
||||
generate_params_if_all_greedy: bool = False
|
||||
generate_params_if_all_greedy: bool = False,
|
||||
) -> "TPUSupportedSamplingMetadata":
|
||||
"""
|
||||
Copy sampling tensors slices from `input_batch` to on device tensors.
|
||||
|
||||
`InputBatch._make_sampling_metadata` causes recompilation on XLA as it
|
||||
slices dynamic shapes on device tensors. This impl moves the dynamic
|
||||
`InputBatch._make_sampling_metadata` causes recompilation on XLA as it
|
||||
slices dynamic shapes on device tensors. This impl moves the dynamic
|
||||
ops to CPU and produces tensors of fixed `padded_num_reqs` size.
|
||||
|
||||
Args:
|
||||
@@ -87,11 +85,11 @@ class TPUSupportedSamplingMetadata:
|
||||
we want to pre-compile a graph with sampling parameters, even if
|
||||
they are not strictly needed for greedy decoding.
|
||||
"""
|
||||
needs_logprobs = input_batch.max_num_logprobs>0 if \
|
||||
input_batch.max_num_logprobs else False
|
||||
needs_logprobs = (
|
||||
input_batch.max_num_logprobs > 0 if input_batch.max_num_logprobs else False
|
||||
)
|
||||
# Early return to avoid unnecessary cpu to tpu copy
|
||||
if (input_batch.all_greedy is True
|
||||
and generate_params_if_all_greedy is False):
|
||||
if input_batch.all_greedy is True and generate_params_if_all_greedy is False:
|
||||
return cls(all_greedy=True, logprobs=needs_logprobs)
|
||||
|
||||
num_reqs = input_batch.num_reqs
|
||||
@@ -100,25 +98,22 @@ class TPUSupportedSamplingMetadata:
|
||||
# Pad value is the default one.
|
||||
cpu_tensor[num_reqs:padded_num_reqs] = fill_val
|
||||
|
||||
fill_slice(input_batch.temperature_cpu_tensor,
|
||||
DEFAULT_SAMPLING_PARAMS["temperature"])
|
||||
fill_slice(input_batch.min_p_cpu_tensor,
|
||||
DEFAULT_SAMPLING_PARAMS["min_p"])
|
||||
fill_slice(input_batch.top_k_cpu_tensor,
|
||||
DEFAULT_SAMPLING_PARAMS["top_k"])
|
||||
fill_slice(input_batch.top_p_cpu_tensor,
|
||||
DEFAULT_SAMPLING_PARAMS["top_p"])
|
||||
fill_slice(
|
||||
input_batch.temperature_cpu_tensor, DEFAULT_SAMPLING_PARAMS["temperature"]
|
||||
)
|
||||
fill_slice(input_batch.min_p_cpu_tensor, DEFAULT_SAMPLING_PARAMS["min_p"])
|
||||
fill_slice(input_batch.top_k_cpu_tensor, DEFAULT_SAMPLING_PARAMS["top_k"])
|
||||
fill_slice(input_batch.top_p_cpu_tensor, DEFAULT_SAMPLING_PARAMS["top_p"])
|
||||
|
||||
# Slice persistent device tensors to a fixed pre-compiled padded shape.
|
||||
return cls(
|
||||
temperature=input_batch.temperature_cpu_tensor[:padded_num_reqs].
|
||||
to(xla_device),
|
||||
temperature=input_batch.temperature_cpu_tensor[:padded_num_reqs].to(
|
||||
xla_device
|
||||
),
|
||||
all_greedy=input_batch.all_greedy,
|
||||
# TODO enable more and avoid returning None values
|
||||
top_p=input_batch.top_p_cpu_tensor[:padded_num_reqs].to(
|
||||
xla_device),
|
||||
top_k=input_batch.top_k_cpu_tensor[:padded_num_reqs].to(
|
||||
xla_device),
|
||||
min_p=input_batch.min_p_cpu_tensor[:padded_num_reqs].to(
|
||||
xla_device),
|
||||
logprobs=needs_logprobs)
|
||||
top_p=input_batch.top_p_cpu_tensor[:padded_num_reqs].to(xla_device),
|
||||
top_k=input_batch.top_k_cpu_tensor[:padded_num_reqs].to(xla_device),
|
||||
min_p=input_batch.min_p_cpu_tensor[:padded_num_reqs].to(xla_device),
|
||||
logprobs=needs_logprobs,
|
||||
)
|
||||
|
||||
@@ -14,7 +14,6 @@ _SAMPLING_EPS = 1e-5
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
# TODO(houseroad): Add support for logprobs_mode.
|
||||
super().__init__()
|
||||
@@ -35,7 +34,8 @@ class Sampler(nn.Module):
|
||||
# [num_requests, 1], where each row represents one generated
|
||||
# token per request.
|
||||
sampled_token_ids=sampled.unsqueeze(-1),
|
||||
logprobs_tensors=None)
|
||||
logprobs_tensors=None,
|
||||
)
|
||||
return sampler_output
|
||||
|
||||
def apply_temperature(
|
||||
@@ -73,11 +73,13 @@ class Sampler(nn.Module):
|
||||
|
||||
# Random sample.
|
||||
probs = logits.softmax(dim=-1, dtype=torch.float32)
|
||||
random_sampled = self.random_sample(probs,
|
||||
sampling_metadata.generators)
|
||||
random_sampled = self.random_sample(probs, sampling_metadata.generators)
|
||||
|
||||
sampled = torch.where(sampling_metadata.temperature < _SAMPLING_EPS,
|
||||
greedy_sampled, random_sampled)
|
||||
sampled = torch.where(
|
||||
sampling_metadata.temperature < _SAMPLING_EPS,
|
||||
greedy_sampled,
|
||||
random_sampled,
|
||||
)
|
||||
return sampled
|
||||
|
||||
def compute_logprobs(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
@@ -107,9 +109,7 @@ class Sampler(nn.Module):
|
||||
Sampled token rank tensor, (num tokens)
|
||||
"""
|
||||
# Find the topK values.
|
||||
topk_logprobs, topk_indices = torch.topk(logprobs,
|
||||
num_logprobs,
|
||||
dim=-1)
|
||||
topk_logprobs, topk_indices = torch.topk(logprobs, num_logprobs, dim=-1)
|
||||
|
||||
# Get with the logprob of the prompt or sampled token.
|
||||
token_ids = token_ids.unsqueeze(-1)
|
||||
@@ -138,9 +138,7 @@ class Sampler(nn.Module):
|
||||
# Convert logits to probability distribution
|
||||
probability_values = torch.nn.functional.softmax(logits, dim=-1)
|
||||
# Calculate maximum probabilities per sequence
|
||||
max_probabilities = torch.amax(probability_values,
|
||||
dim=-1,
|
||||
keepdim=True)
|
||||
max_probabilities = torch.amax(probability_values, dim=-1, keepdim=True)
|
||||
# Reshape min_p for broadcasting
|
||||
adjusted_min_p = min_p.unsqueeze(1) * max_probabilities
|
||||
# Identify valid tokens using threshold comparison
|
||||
|
||||
Reference in New Issue
Block a user