[V1] Decouple GPU and TPU InputBatch (#19778)

Signed-off-by: Andrew Feldman <afeldman@redhat.com>
This commit is contained in:
afeldman-nm
2025-06-18 06:38:13 +00:00
committed by GitHub
parent eccdc8318c
commit 19a53b2783
5 changed files with 597 additions and 4 deletions
+6 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Datastructures defining an input batch
# Datastructures defining a GPU input batch
from dataclasses import dataclass
from typing import Optional, cast
@@ -453,6 +453,11 @@ class InputBatch:
self.block_table.swap_row(i1, i2)
def condense(self, empty_req_indices: list[int]) -> None:
"""Move non-empty requests down into lower, empty indices.
Args:
empty_req_indices: empty batch indices, sorted descending.
"""
num_reqs = self.num_reqs
if num_reqs == 0:
# The batched states are empty.