[Feature] limit thinking tokens (hard limit) (#20859)

Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com>
Signed-off-by: Sungjae Lee <sung-jae.lee@navercorp.com>
Signed-off-by: Chauncey <chaunceyjiang@gmail.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Sungjae Lee
2026-03-24 09:53:07 -07:00
committed by GitHub
co-authored by Chauncey Harry Mellor
parent 8de5261e69
commit 4731884796
13 changed files with 702 additions and 12 deletions
+5
View File
@@ -281,6 +281,8 @@ class SamplingParams(
_bad_words_token_ids: list[list[int]] | None = None
skip_reading_prefix_cache: bool | None = None
thinking_token_budget: int | None = None
"""Maximum number of tokens allowed for thinking operations."""
repetition_detection: RepetitionDetectionParams | None = None
"""Parameters for detecting repetitive N-gram patterns in output tokens.
@@ -304,6 +306,7 @@ class SamplingParams(
stop: str | list[str] | None = None,
stop_token_ids: list[int] | None = None,
bad_words: list[str] | None = None,
thinking_token_budget: int | None = None,
include_stop_str_in_output: bool = False,
ignore_eos: bool = False,
max_tokens: int | None = 16,
@@ -344,6 +347,7 @@ class SamplingParams(
stop=stop,
stop_token_ids=stop_token_ids,
bad_words=bad_words,
thinking_token_budget=thinking_token_budget,
include_stop_str_in_output=include_stop_str_in_output,
ignore_eos=ignore_eos,
max_tokens=max_tokens,
@@ -858,6 +862,7 @@ class SamplingParams(
f"stop={self.stop}, "
f"stop_token_ids={self.stop_token_ids}, "
f"bad_words={self.bad_words}, "
f"thinking_token_budget={self.thinking_token_budget}, "
f"include_stop_str_in_output={self.include_stop_str_in_output}, "
f"ignore_eos={self.ignore_eos}, "
f"max_tokens={self.max_tokens}, "