[TPU][V1] Disable per-request seed/Generator (#16172)

Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2025-04-10 17:05:44 -04:00
committed by GitHub
parent 7cd0bd7212
commit 3cc9af88ff
4 changed files with 24 additions and 18 deletions
+10 -6
View File
@@ -33,10 +33,6 @@ class TPUSupportedSamplingMetadata:
# Greedy sampling flag for compiling single xla graph.
all_greedy: bool = True
# Generator not supported by xla
generators: dict[int,
torch.Generator] = field(default_factory=lambda: dict())
# unsupported, you need to return an extra tensor of static size BxV
max_num_logprobs = None
@@ -57,6 +53,15 @@ class TPUSupportedSamplingMetadata:
allowed_token_ids_mask = None
bad_words_token_ids = None
# Generator not supported by xla
_generators: dict[int,
torch.Generator] = field(default_factory=lambda: dict())
@property
def generators(self) -> dict[int, torch.Generator]:
# Generator not supported by torch/xla. This field must be immutable.
return self._generators
@classmethod
def from_input_batch(
cls,
@@ -109,5 +114,4 @@ class TPUSupportedSamplingMetadata:
top_p=None, # input_batch.top_p[:padded_num_reqs],
top_k=None, # input_batch.top_k[:padded_num_reqs],
min_p=input_batch.min_p_cpu_tensor[:padded_num_reqs].to(
xla_device),
generators=input_batch.generators)
xla_device))