[Core][V1][TPU] Enable structured decoding on TPU V1 (#16499)

Signed-off-by: Chenyaaang <chenyangli@google.com>
This commit is contained in:
Chenyaaang
2025-04-22 18:05:23 -06:00
committed by GitHub
parent 5175b884f7
commit 83d933718c
5 changed files with 158 additions and 31 deletions
+5 -2
View File
@@ -23,7 +23,7 @@ def test_sampler_different(model_name: str):
different results.
"""
llm = LLM(model_name,
enforce_eager=False,
enforce_eager=True,
max_num_seqs=1,
max_model_len=512,
max_num_batched_tokens=512)
@@ -57,4 +57,7 @@ def test_sampler_different(model_name: str):
# Make sure first two reqs have the same K/P
sampling_params[0] = sampling_params[1]
output = llm.generate(p, sampling_params)
assert output[0].outputs[0].text == output[1].outputs[0].text
# There are natural numerical instabilities that make it difficult
# to have deterministic results over many tokens, tests the first ~20
# tokens match.
assert output[0].outputs[0].text[:20] == output[1].outputs[0].text[:20]