From 7dbc3d62c388df67d2764f28c90c51cc11ff0329 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 14 Sep 2026 21:29:15 -0700 Subject: [PATCH] [Cleanup] Remove vestigial `tpu_input_batch.py` (#56898) Signed-off-by: Nick Hill --- .../v1/e2e/general/test_mamba_prefix_cache.py | 7 +- vllm/v1/worker/lora_model_runner_mixin.py | 6 +- vllm/v1/worker/mamba_utils.py | 11 +- vllm/v1/worker/tpu_input_batch.py | 559 ------------------ 4 files changed, 9 insertions(+), 574 deletions(-) delete mode 100644 vllm/v1/worker/tpu_input_batch.py diff --git a/tests/v1/e2e/general/test_mamba_prefix_cache.py b/tests/v1/e2e/general/test_mamba_prefix_cache.py index aa1f730afb..e388b73468 100644 --- a/tests/v1/e2e/general/test_mamba_prefix_cache.py +++ b/tests/v1/e2e/general/test_mamba_prefix_cache.py @@ -31,9 +31,8 @@ from vllm.v1.request import Request from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.spec_decode.metadata import SpecDecodeMetadata from vllm.v1.worker import mamba_utils -from vllm.v1.worker.gpu_input_batch import CachedRequestState +from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch from vllm.v1.worker.gpu_model_runner import GPUModelRunner -from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch from vllm.v1.worker.mamba_utils import get_mamba_groups @@ -315,7 +314,7 @@ def get_fake_process_mamba_fn( action: tuple[int, int], kv_cache_config: KVCacheConfig, forward_context: dict[str, Any], - input_batch: GPUInputBatch, + input_batch: InputBatch, ): assert copy_info is not None if action == (-1, -1): @@ -371,7 +370,7 @@ def get_fake_process_mamba_fn( kv_cache_config: KVCacheConfig, cache_config: CacheConfig, mamba_state_idx: dict[str, int], - input_batch: GPUInputBatch, + input_batch: InputBatch, requests: dict[str, CachedRequestState], forward_context: dict[str, Any], mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], diff --git a/vllm/v1/worker/lora_model_runner_mixin.py b/vllm/v1/worker/lora_model_runner_mixin.py index 27e850b66a..5e7f1f70c5 100644 --- a/vllm/v1/worker/lora_model_runner_mixin.py +++ b/vllm/v1/worker/lora_model_runner_mixin.py @@ -6,7 +6,6 @@ Define LoRA functionality mixin for model runners. from collections.abc import Callable from contextlib import contextmanager -from typing import TypeAlias import numpy as np import torch @@ -19,10 +18,7 @@ from vllm.lora.layers import LoRAMapping, LoRAMappingType from vllm.lora.request import LoRARequest from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager from vllm.model_executor.models import supports_lora -from vllm.v1.worker.gpu_input_batch import InputBatch as GPUInputBatch -from vllm.v1.worker.tpu_input_batch import InputBatch as TPUInputBatch - -InputBatch: TypeAlias = TPUInputBatch | GPUInputBatch +from vllm.v1.worker.gpu_input_batch import InputBatch logger = init_logger(__name__) diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 724c714057..1deb89bda6 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -26,8 +26,7 @@ from vllm.v1.kv_cache_interface import ( UniformTypeKVCacheSpecs, ) from vllm.v1.utils import CpuGpuBuffer -from vllm.v1.worker.gpu_input_batch import CachedRequestState -from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch +from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch logger = init_logger(__name__) @@ -1436,7 +1435,7 @@ def preprocess_mamba( kv_cache_config: KVCacheConfig, cache_config: CacheConfig, mamba_state_idx: dict[str, int], - input_batch: GPUInputBatch, + input_batch: InputBatch, requests: dict[str, CachedRequestState], forward_context: dict[str, Any], mamba_state_copy_funcs: MambaStateCopyFuncsByType, @@ -1541,7 +1540,7 @@ def preprocess_mamba( def postprocess_mamba_all( scheduler_output: SchedulerOutput, kv_cache_config: KVCacheConfig, - input_batch: GPUInputBatch, + input_batch: InputBatch, requests: dict[str, CachedRequestState], mamba_state_idx: dict[str, int], num_spec_tokens: int, @@ -1572,7 +1571,7 @@ def postprocess_mamba_all( def preprocess_mamba_all_specdec( scheduler_output: SchedulerOutput, - input_batch: GPUInputBatch, + input_batch: InputBatch, mamba_state_idx: dict[str, int], num_reqs: int, prev_last_scheduled_idx_buf: CpuGpuBuffer, @@ -1591,7 +1590,7 @@ def postprocess_mamba_align_gpu( num_reqs: int, num_accepted_tokens_gpu: torch.Tensor, num_accepted_tokens_cpu_tensor: torch.Tensor, - input_batch: GPUInputBatch, + input_batch: InputBatch, kv_cache_config: KVCacheConfig, forward_context: dict[str, Any], mamba_state_copy_funcs: MambaStateCopyFuncsByType, diff --git a/vllm/v1/worker/tpu_input_batch.py b/vllm/v1/worker/tpu_input_batch.py deleted file mode 100644 index c16156a5e3..0000000000 --- a/vllm/v1/worker/tpu_input_batch.py +++ /dev/null @@ -1,559 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Datastructures defining a TPU input batch - -from typing import cast - -import numpy as np -import torch - -from vllm.lora.request import LoRARequest -from vllm.sampling_params import SamplingType -from vllm.utils import length_from_prompt_token_ids_or_embeds -from vllm.utils.collection_utils import swap_dict_values -from vllm.v1.outputs import LogprobsTensors -from vllm.v1.worker.block_table import MultiGroupBlockTable -from vllm.v1.worker.gpu_input_batch import CachedRequestState - -_SAMPLING_EPS = 1e-5 - - -class InputBatch: - def __init__( - self, - max_num_reqs: int, - max_model_len: int, - max_num_batched_tokens: int, - device: torch.device, - pin_memory: bool, - vocab_size: int, - block_sizes: list[int], # The block_size of each kv cache group - kernel_block_sizes: list[int], - max_num_blocks_per_req: list[int], - ): - self.max_num_reqs = max_num_reqs - self.max_model_len = max_model_len - self.max_num_batched_tokens = max_num_batched_tokens - self.device = device - self.pin_memory = pin_memory - self.vocab_size = vocab_size - - self._req_ids: list[str | None] = [] - self.req_id_to_index: dict[str, int] = {} - - # TODO(woosuk): This buffer could be too large if max_model_len is big. - # Find a way to reduce the CPU memory usage. - # This buffer is not directly transferred to the GPU, so it does not - # need to be pinned. - self.token_ids_cpu_tensor = torch.zeros( - (max_num_reqs, max_model_len), - device="cpu", - dtype=torch.int32, - pin_memory=False, - ) - self.token_ids_cpu = self.token_ids_cpu_tensor.numpy() - self.num_tokens_no_spec = np.zeros(max_num_reqs, dtype=np.int32) - self.num_prompt_tokens = np.zeros(max_num_reqs, dtype=np.int32) - self.num_computed_tokens_cpu_tensor = torch.zeros( - (max_num_reqs,), - device="cpu", - dtype=torch.int32, - pin_memory=pin_memory, - ) - self.num_computed_tokens_cpu = self.num_computed_tokens_cpu_tensor.numpy() - - # Block table. - self.block_table = MultiGroupBlockTable( - max_num_reqs=max_num_reqs, - max_num_batched_tokens=max_num_batched_tokens, - pin_memory=pin_memory, - device=device, - block_sizes=block_sizes, - kernel_block_sizes=kernel_block_sizes, - max_num_blocks=max_num_blocks_per_req, - ) - - # Sampling-related. - self.temperature = torch.empty( - (max_num_reqs,), dtype=torch.float32, device=device - ) - self.temperature_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory - ) - self.temperature_cpu = self.temperature_cpu_tensor.numpy() - self.greedy_reqs: set[str] = set() - self.random_reqs: set[str] = set() - - self.top_p = torch.empty((max_num_reqs,), dtype=torch.float32, device=device) - self.top_p_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory - ) - self.top_p_cpu = self.top_p_cpu_tensor.numpy() - self.top_p_reqs: set[str] = set() - - self.top_k = torch.empty((max_num_reqs,), dtype=torch.int32, device=device) - self.top_k_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.int32, device="cpu", pin_memory=pin_memory - ) - self.top_k_cpu = self.top_k_cpu_tensor.numpy() - self.top_k_reqs: set[str] = set() - - self.min_p = torch.empty((max_num_reqs,), dtype=torch.float32, device=device) - self.min_p_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory - ) - self.min_p_cpu = self.min_p_cpu_tensor.numpy() - self.min_p_reqs: set[str] = set() - - # Frequency penalty related data structures - self.frequency_penalties = torch.empty( - (max_num_reqs,), dtype=torch.float, device=device - ) - self.frequency_penalties_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory - ) - self.frequency_penalties_cpu = self.frequency_penalties_cpu_tensor.numpy() - self.frequency_penalties_reqs: set[str] = set() - - # Presence penalty related data structures - self.presence_penalties = torch.empty( - (max_num_reqs,), dtype=torch.float, device=device - ) - self.presence_penalties_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory - ) - self.presence_penalties_cpu = self.presence_penalties_cpu_tensor.numpy() - self.presence_penalties_reqs: set[str] = set() - - # Repetition penalty related data structures - self.repetition_penalties = torch.empty( - (max_num_reqs,), dtype=torch.float, device=device - ) - self.repetition_penalties_cpu_tensor = torch.empty( - (max_num_reqs,), dtype=torch.float, device="cpu", pin_memory=pin_memory - ) - self.repetition_penalties_cpu = self.repetition_penalties_cpu_tensor.numpy() - self.repetition_penalties_reqs: set[str] = set() - - # req_index -> (min_tokens, stop_token_ids) - self.min_tokens: dict[int, tuple[int, set[int]]] = {} - - # lora related - self.request_lora_mapping = np.zeros((self.max_num_reqs,), dtype=np.int64) - self.lora_id_to_request_ids: dict[int, set[str]] = {} - self.lora_id_to_lora_request: dict[int, LoRARequest] = {} - - # req_index -> generator - # NOTE(woosuk): The indices of the requests that do not have their own - # generator should not be included in the dictionary. - self.generators: dict[int, torch.Generator] = {} - - self.num_logprobs: dict[str, int] = {} - - # To accumulate prompt logprobs tensor chunks across prefill steps. - self.in_progress_prompt_logprobs_cpu: dict[str, LogprobsTensors] = {} - - self.logit_bias: list[dict[int, float] | None] = [None] * max_num_reqs - self.has_allowed_token_ids: set[str] = set() - # NOTE(lufang): In the mask tensor, if the corresponding token allowed, - # the value is False. Since we use masked_fill_ to set -inf. - self.allowed_token_ids_mask: torch.Tensor | None = None - self.allowed_token_ids_mask_cpu_tensor: torch.Tensor | None = None - - # req_index -> bad_words_token_ids - self.bad_words_token_ids: dict[int, list[list[int]]] = {} - - self.req_output_token_ids: list[list[int] | None] = [] - - @property - def req_ids(self) -> list[str]: - # None elements should only be present transiently - # while performing state updates to the batch. - return cast(list[str], self._req_ids) - - def add_request( - self, - request: "CachedRequestState", - req_index: int | None = None, - ) -> None: - if req_index is None: - req_index = self.num_reqs - assert req_index < self.max_num_reqs - - req_id = request.req_id - if req_index == len(self._req_ids): - self._req_ids.append(req_id) - self.req_output_token_ids.append(request.output_token_ids) - else: - self._req_ids[req_index] = req_id - self.req_output_token_ids[req_index] = request.output_token_ids - - self.req_id_to_index[req_id] = req_index - - # Copy the prompt token ids and output token ids. - num_prompt_tokens = length_from_prompt_token_ids_or_embeds( - request.prompt_token_ids, request.prompt_embeds - ) - # TODO: copy prompt_embeds - self.num_prompt_tokens[req_index] = num_prompt_tokens - self.token_ids_cpu[req_index, :num_prompt_tokens] = request.prompt_token_ids - start_idx = num_prompt_tokens - end_idx = start_idx + len(request.output_token_ids) - self.token_ids_cpu[req_index, start_idx:end_idx] = request.output_token_ids - # Number of tokens without spec decode tokens. - self.num_tokens_no_spec[req_index] = request.num_tokens - - self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens - self.block_table.add_row(request.block_ids, req_index) - - sampling_params = request.sampling_params - assert sampling_params is not None, "pooling requests not supported yet" - if sampling_params.sampling_type == SamplingType.GREEDY: - # Should avoid division by zero later when apply_temperature. - self.temperature_cpu[req_index] = 0.0 - self.greedy_reqs.add(req_id) - else: - self.temperature_cpu[req_index] = sampling_params.temperature - self.random_reqs.add(req_id) - - self.top_p_cpu[req_index] = sampling_params.top_p - if sampling_params.top_p < 1: - self.top_p_reqs.add(req_id) - top_k = sampling_params.top_k - if 0 < top_k < self.vocab_size: - self.top_k_reqs.add(req_id) - else: - top_k = self.vocab_size - self.top_k_cpu[req_index] = top_k - self.min_p_cpu[req_index] = sampling_params.min_p - self.frequency_penalties_cpu[req_index] = sampling_params.frequency_penalty - if sampling_params.min_p > _SAMPLING_EPS: - self.min_p_reqs.add(req_id) - if sampling_params.frequency_penalty != 0.0: - self.frequency_penalties_reqs.add(req_id) - self.presence_penalties_cpu[req_index] = sampling_params.presence_penalty - if sampling_params.presence_penalty != 0.0: - self.presence_penalties_reqs.add(req_id) - self.repetition_penalties_cpu[req_index] = sampling_params.repetition_penalty - if sampling_params.repetition_penalty != 1.0: - self.repetition_penalties_reqs.add(req_id) - if sampling_params.min_tokens: - self.min_tokens[req_index] = ( - sampling_params.min_tokens, - sampling_params.all_stop_token_ids, - ) - - # NOTE(woosuk): self.generators should not include the requests that - # do not have their own generator. - if request.generator is not None: - self.generators[req_index] = request.generator - - if sampling_params.logprobs is not None: - self.num_logprobs[req_id] = sampling_params.logprobs - if sampling_params.logit_bias is not None: - self.logit_bias[req_index] = sampling_params.logit_bias - - if sampling_params.allowed_token_ids: - self.has_allowed_token_ids.add(req_id) - if self.allowed_token_ids_mask_cpu_tensor is None: - # Lazy allocation for this tensor, which can be large. - # False means we don't fill with -inf. - self.allowed_token_ids_mask = torch.zeros( - self.max_num_reqs, - self.vocab_size, - dtype=torch.bool, - device=self.device, - ) - self.allowed_token_ids_mask_cpu_tensor = torch.zeros( - self.max_num_reqs, self.vocab_size, dtype=torch.bool, device="cpu" - ) - self.allowed_token_ids_mask_cpu_tensor[req_index] = True - # False means we don't fill with -inf. - self.allowed_token_ids_mask_cpu_tensor[req_index][ - sampling_params.allowed_token_ids - ] = False - - if sampling_params.bad_words_token_ids: - self.bad_words_token_ids[req_index] = sampling_params.bad_words_token_ids - - # Add request lora ID - if request.lora_request: - lora_id = request.lora_request.lora_int_id - if lora_id not in self.lora_id_to_request_ids: - self.lora_id_to_request_ids[lora_id] = set() - - self.request_lora_mapping[req_index] = lora_id - self.lora_id_to_request_ids[lora_id].add(request.req_id) - self.lora_id_to_lora_request[lora_id] = request.lora_request - else: - # No LoRA - self.request_lora_mapping[req_index] = 0 - - def remove_request(self, req_id: str) -> int | None: - """This method must always be followed by a call to condense().""" - - req_index = self.req_id_to_index.pop(req_id, None) - if req_index is None: - return None - self._req_ids[req_index] = None - self.req_output_token_ids[req_index] = None - - self.greedy_reqs.discard(req_id) - self.random_reqs.discard(req_id) - self.top_p_reqs.discard(req_id) - self.top_k_reqs.discard(req_id) - self.min_p_reqs.discard(req_id) - self.min_tokens.pop(req_index, None) - self.frequency_penalties_reqs.discard(req_id) - self.presence_penalties_reqs.discard(req_id) - self.repetition_penalties_reqs.discard(req_id) - self.generators.pop(req_index, None) - self.num_logprobs.pop(req_id, None) - self.in_progress_prompt_logprobs_cpu.pop(req_id, None) - - # LoRA - lora_id = self.request_lora_mapping[req_index] - if lora_id != 0: - self.lora_id_to_request_ids[lora_id].discard(req_id) - if len(self.lora_id_to_request_ids[lora_id]) == 0: - self.lora_id_to_request_ids.pop(lora_id) - self.lora_id_to_lora_request.pop(lora_id) - self.request_lora_mapping[req_index] = 0 - - self.logit_bias[req_index] = None - self.has_allowed_token_ids.discard(req_id) - if self.allowed_token_ids_mask_cpu_tensor is not None: - # False means we don't fill with -inf. - self.allowed_token_ids_mask_cpu_tensor[req_index].fill_(False) - self.bad_words_token_ids.pop(req_index, None) - return req_index - - def swap_states(self, i1: int, i2: int) -> None: - old_id_i1 = self._req_ids[i1] - old_id_i2 = self._req_ids[i2] - self._req_ids[i1], self._req_ids[i2] = self._req_ids[i2], self._req_ids[i1] # noqa - self.req_output_token_ids[i1], self.req_output_token_ids[i2] = ( - self.req_output_token_ids[i2], - self.req_output_token_ids[i1], - ) - assert old_id_i1 is not None and old_id_i2 is not None - self.req_id_to_index[old_id_i1], self.req_id_to_index[old_id_i2] = ( - self.req_id_to_index[old_id_i2], - self.req_id_to_index[old_id_i1], - ) - self.num_tokens_no_spec[i1], self.num_tokens_no_spec[i2] = ( - self.num_tokens_no_spec[i2], - self.num_tokens_no_spec[i1], - ) - self.num_prompt_tokens[i1], self.num_prompt_tokens[i2] = ( - self.num_prompt_tokens[i2], - self.num_prompt_tokens[i1], - ) - self.num_computed_tokens_cpu[i1], self.num_computed_tokens_cpu[i2] = ( - self.num_computed_tokens_cpu[i2], - self.num_computed_tokens_cpu[i1], - ) - self.temperature_cpu[i1], self.temperature_cpu[i2] = ( - self.temperature_cpu[i2], - self.temperature_cpu[i1], - ) - self.top_p_cpu[i1], self.top_p_cpu[i2] = self.top_p_cpu[i2], self.top_p_cpu[i1] - self.top_k_cpu[i1], self.top_k_cpu[i2] = self.top_k_cpu[i2], self.top_k_cpu[i1] - self.frequency_penalties_cpu[i1], self.frequency_penalties_cpu[i2] = ( - self.frequency_penalties_cpu[i2], - self.frequency_penalties_cpu[i1], - ) - self.presence_penalties_cpu[i1], self.presence_penalties_cpu[i2] = ( - self.presence_penalties_cpu[i2], - self.presence_penalties_cpu[i1], - ) - self.repetition_penalties_cpu[i1], self.repetition_penalties_cpu[i2] = ( - self.repetition_penalties_cpu[i2], - self.repetition_penalties_cpu[i1], - ) - self.min_p_cpu[i1], self.min_p_cpu[i2] = self.min_p_cpu[i2], self.min_p_cpu[i1] - - # NOTE: the following is unsafe - # self.token_ids_cpu[i1, ...], self.token_ids_cpu[i2, ...], =\ - # self.token_ids_cpu[i2, ...], self.token_ids_cpu[i1, ...] - # instead, we need to temporarily copy the data for one of the indices - n = max(self.num_tokens_no_spec[i1], self.num_tokens_no_spec[i2]) - tmp = self.token_ids_cpu[i1, :n].copy() - self.token_ids_cpu[i1, :n] = self.token_ids_cpu[i2, :n] - self.token_ids_cpu[i2, :n] = tmp - - swap_dict_values(self.generators, i1, i2) - swap_dict_values(self.min_tokens, i1, i2) - swap_dict_values(self.bad_words_token_ids, i1, i2) - - self.request_lora_mapping[i1], self.request_lora_mapping[i2] = ( - self.request_lora_mapping[i2], - self.request_lora_mapping[i1], - ) - self.logit_bias[i1], self.logit_bias[i2] = ( - self.logit_bias[i2], - self.logit_bias[i1], - ) - - if self.allowed_token_ids_mask_cpu_tensor is not None: - ( - self.allowed_token_ids_mask_cpu_tensor[i1], - self.allowed_token_ids_mask_cpu_tensor[i2], - ) = ( - self.allowed_token_ids_mask_cpu_tensor[i2], - self.allowed_token_ids_mask_cpu_tensor[i1], - ) - self.block_table.swap_row(i1, i2) - - def condense(self, empty_req_indices: list[int]) -> None: - """Move non-empty requests down into lower, empty indices. - - Args: - empty_req_indices: empty batch indices, sorted descending. - """ - num_reqs = self.num_reqs - if num_reqs == 0: - # The batched states are empty. - self._req_ids.clear() - self.req_output_token_ids.clear() - return - - # NOTE(woosuk): This function assumes that the empty_req_indices - # is sorted in descending order. - last_req_index = num_reqs + len(empty_req_indices) - 1 - while empty_req_indices: - # Find the largest non-empty index. - while last_req_index in empty_req_indices: - last_req_index -= 1 - - # Find the smallest empty index. - empty_index = empty_req_indices.pop() - if empty_index >= last_req_index: - break - - # Swap the states. - req_id = self._req_ids[last_req_index] - output_token_ids = self.req_output_token_ids[last_req_index] - assert req_id is not None - self._req_ids[empty_index] = req_id - self._req_ids[last_req_index] = None - self.req_output_token_ids[empty_index] = output_token_ids - self.req_output_token_ids[last_req_index] = None - self.req_id_to_index[req_id] = empty_index - - num_tokens = self.num_tokens_no_spec[last_req_index] - self.token_ids_cpu[empty_index, :num_tokens] = self.token_ids_cpu[ - last_req_index, :num_tokens - ] - self.num_tokens_no_spec[empty_index] = self.num_tokens_no_spec[ - last_req_index - ] - self.num_prompt_tokens[empty_index] = self.num_prompt_tokens[last_req_index] - self.num_computed_tokens_cpu[empty_index] = self.num_computed_tokens_cpu[ - last_req_index - ] - self.block_table.move_row(last_req_index, empty_index) - self.temperature_cpu[empty_index] = self.temperature_cpu[last_req_index] - self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index] - self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index] - self.frequency_penalties_cpu[empty_index] = self.frequency_penalties_cpu[ - last_req_index - ] - self.presence_penalties_cpu[empty_index] = self.presence_penalties_cpu[ - last_req_index - ] - self.repetition_penalties_cpu[empty_index] = self.repetition_penalties_cpu[ - last_req_index - ] - self.min_p_cpu[empty_index] = self.min_p_cpu[last_req_index] - generator = self.generators.pop(last_req_index, None) - if generator is not None: - self.generators[empty_index] = generator - - min_token = self.min_tokens.pop(last_req_index, None) - if min_token is not None: - self.min_tokens[empty_index] = min_token - - self.request_lora_mapping[empty_index] = self.request_lora_mapping[ - last_req_index - ] - - self.logit_bias[empty_index] = self.logit_bias[last_req_index] - - if self.allowed_token_ids_mask_cpu_tensor is not None: - self.allowed_token_ids_mask_cpu_tensor[empty_index] = ( - self.allowed_token_ids_mask_cpu_tensor[last_req_index] - ) - - bad_words_token_ids = self.bad_words_token_ids.pop(last_req_index, None) - if bad_words_token_ids is not None: - self.bad_words_token_ids[empty_index] = bad_words_token_ids - # Decrement last_req_index since it is now empty. - last_req_index -= 1 - - # Trim lists to the batch size. - del self._req_ids[self.num_reqs :] - del self.req_output_token_ids[self.num_reqs :] - - def make_lora_inputs( - self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray - ) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]: - """ - Given the num_scheduled_tokens for each request in the batch, return - datastructures used to activate the current LoRAs. - Returns: - 1. prompt_lora_mapping: A tuple of size self.num_reqs where, - prompt_lora_mapping[i] is the LoRA id to use for the ith prompt. - 2. token_lora_mapping: A tuple of size np.sum(num_scheduled_tokens) - where, token_lora_mapping[i] is the LoRA id to use for ith token. - 3. lora_requests: Set of relevant LoRA requests. - """ - - req_lora_mapping = self.request_lora_mapping[: self.num_reqs] - prompt_lora_mapping = tuple(req_lora_mapping) - token_lora_mapping = tuple(req_lora_mapping.repeat(num_scheduled_tokens)) - active_lora_requests: set[LoRARequest] = set( - self.lora_id_to_lora_request.values() - ) - - return prompt_lora_mapping, token_lora_mapping, active_lora_requests - - @property - def num_reqs(self) -> int: - return len(self.req_id_to_index) - - @property - def all_greedy(self) -> bool: - return len(self.random_reqs) == 0 - - @property - def all_random(self) -> bool: - return len(self.greedy_reqs) == 0 - - @property - def no_top_p(self) -> bool: - return len(self.top_p_reqs) == 0 - - @property - def no_top_k(self) -> bool: - return len(self.top_k_reqs) == 0 - - @property - def no_min_p(self) -> bool: - return len(self.min_p_reqs) == 0 - - @property - def no_penalties(self) -> bool: - return ( - len(self.presence_penalties_reqs) == 0 - and len(self.frequency_penalties_reqs) == 0 - and len(self.repetition_penalties_reqs) == 0 - ) - - @property - def max_num_logprobs(self) -> int | None: - return max(self.num_logprobs.values()) if self.num_logprobs else None - - @property - def no_allowed_token_ids(self) -> bool: - return len(self.has_allowed_token_ids) == 0