[Scheduler] Soften Long Prefill Tokens Threshhold (#57951)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
Robert Shaw
2026-09-21 17:19:42 -07:00
committed by GitHub
co-authored by Robert Shaw
parent 78d7836525
commit c64b15cde5
5 changed files with 52 additions and 11 deletions
@@ -77,13 +77,13 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
chunk_size = 10
# Set chunking parameters to force chunked prefill
# Note: Chunked prefill is automatically handled by vLLM
# internally based on the model size and prompt
# Cap the token budget to force chunked prefill; the
# long_prefill_token_threshold is not applied to a lone request.
with vllm_runner(
model_id,
runner="pooling",
long_prefill_token_threshold=chunk_size,
max_num_batched_tokens=chunk_size,
max_num_seqs=1,
tensor_parallel_size=1,
enforce_eager=True,
enable_chunked_prefill=True,