[Core][V1][TPU] Enable structured decoding on TPU V1 (#16499)
Signed-off-by: Chenyaaang <chenyangli@google.com>
This commit is contained in:
@@ -23,7 +23,7 @@ def test_sampler_different(model_name: str):
|
||||
different results.
|
||||
"""
|
||||
llm = LLM(model_name,
|
||||
enforce_eager=False,
|
||||
enforce_eager=True,
|
||||
max_num_seqs=1,
|
||||
max_model_len=512,
|
||||
max_num_batched_tokens=512)
|
||||
@@ -57,4 +57,7 @@ def test_sampler_different(model_name: str):
|
||||
# Make sure first two reqs have the same K/P
|
||||
sampling_params[0] = sampling_params[1]
|
||||
output = llm.generate(p, sampling_params)
|
||||
assert output[0].outputs[0].text == output[1].outputs[0].text
|
||||
# There are natural numerical instabilities that make it difficult
|
||||
# to have deterministic results over many tokens, tests the first ~20
|
||||
# tokens match.
|
||||
assert output[0].outputs[0].text[:20] == output[1].outputs[0].text[:20]
|
||||
|
||||
Reference in New Issue
Block a user