[Hardware][TPU] Fix the recompiling issue in logits processor after warmup (#14510)

Signed-off-by: Chengji Yao <chengjiyao@google.com>
This commit is contained in:
Chengji Yao
2025-03-09 05:44:39 -04:00
committed by GitHub
parent fb16eea48b
commit 212007b168
2 changed files with 41 additions and 10 deletions
+3 -1
View File
@@ -21,7 +21,9 @@ sampling_params = SamplingParams(temperature=0.7,
# Set `enforce_eager=True` to avoid ahead-of-time compilation.
# In real workloads, `enforace_eager` should be `False`.
llm = LLM(model="google/gemma-2b", enforce_eager=True)
llm = LLM(model="Qwen/Qwen2-1.5B-Instruct",
max_num_batched_tokens=64,
max_num_seqs=4)
outputs = llm.generate(prompts, sampling_params)
for output, answer in zip(outputs, answers):
prompt = output.prompt