[Feature] limit thinking tokens (hard limit) (#20859)
Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com> Signed-off-by: Sungjae Lee <sung-jae.lee@navercorp.com> Signed-off-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Chauncey
Harry Mellor
parent
8de5261e69
commit
4731884796
@@ -281,6 +281,8 @@ class SamplingParams(
|
||||
_bad_words_token_ids: list[list[int]] | None = None
|
||||
|
||||
skip_reading_prefix_cache: bool | None = None
|
||||
thinking_token_budget: int | None = None
|
||||
"""Maximum number of tokens allowed for thinking operations."""
|
||||
|
||||
repetition_detection: RepetitionDetectionParams | None = None
|
||||
"""Parameters for detecting repetitive N-gram patterns in output tokens.
|
||||
@@ -304,6 +306,7 @@ class SamplingParams(
|
||||
stop: str | list[str] | None = None,
|
||||
stop_token_ids: list[int] | None = None,
|
||||
bad_words: list[str] | None = None,
|
||||
thinking_token_budget: int | None = None,
|
||||
include_stop_str_in_output: bool = False,
|
||||
ignore_eos: bool = False,
|
||||
max_tokens: int | None = 16,
|
||||
@@ -344,6 +347,7 @@ class SamplingParams(
|
||||
stop=stop,
|
||||
stop_token_ids=stop_token_ids,
|
||||
bad_words=bad_words,
|
||||
thinking_token_budget=thinking_token_budget,
|
||||
include_stop_str_in_output=include_stop_str_in_output,
|
||||
ignore_eos=ignore_eos,
|
||||
max_tokens=max_tokens,
|
||||
@@ -858,6 +862,7 @@ class SamplingParams(
|
||||
f"stop={self.stop}, "
|
||||
f"stop_token_ids={self.stop_token_ids}, "
|
||||
f"bad_words={self.bad_words}, "
|
||||
f"thinking_token_budget={self.thinking_token_budget}, "
|
||||
f"include_stop_str_in_output={self.include_stop_str_in_output}, "
|
||||
f"ignore_eos={self.ignore_eos}, "
|
||||
f"max_tokens={self.max_tokens}, "
|
||||
|
||||
Reference in New Issue
Block a user