[Core][Engine] only materialize tokens when thinking budget is in req (#47053)
Signed-off-by: walterbm <walter.beller.morales@gmail.com>
This commit is contained in:
@@ -702,11 +702,9 @@ class GPUModelRunner(
|
||||
custom_logitsprocs,
|
||||
),
|
||||
# We currently don't know whether a particular custom logits processor
|
||||
# uses output token ids so we set this conservatively.
|
||||
# ThinkingTokenBudgetLogitsProcessor also needs output token ids to
|
||||
# correctly track think start/end token sequences in async scheduling.
|
||||
logitsprocs_need_output_token_ids=bool(custom_logitsprocs)
|
||||
or self.vllm_config.reasoning_config is not None,
|
||||
# uses output token ids so we set this conservatively. Thinking-budget
|
||||
# tracking is requested dynamically when a budgeted request is in the batch.
|
||||
logitsprocs_need_output_token_ids=bool(custom_logitsprocs),
|
||||
is_pooling_model=self.is_pooling_model,
|
||||
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
|
||||
reasoning_config=self.vllm_config.reasoning_config,
|
||||
|
||||
Reference in New Issue
Block a user