fix(config): reject negative max_logprobs (except -1) and long_prefill_token_threshold (#44070)
Signed-off-by: Chenglun Hu <chenglunhu@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5b4cb69523
commit
af1ee8c475
@@ -213,7 +213,7 @@ class ModelConfig:
|
||||
flexibility."""
|
||||
enable_return_routed_experts: bool = False
|
||||
"""Whether to return routed experts."""
|
||||
max_logprobs: int = 20
|
||||
max_logprobs: int = Field(default=20, ge=-1)
|
||||
"""Maximum number of log probabilities to return when `logprobs` is
|
||||
specified in `SamplingParams`. The default value comes the default for the
|
||||
OpenAI Chat Completions API. -1 means no cap, i.e. all (output_length *
|
||||
|
||||
@@ -77,7 +77,7 @@ class SchedulerConfig:
|
||||
this less than max_num_partial_prefills will allow shorter prompts to jump
|
||||
the queue in front of longer prompts in some cases, improving latency."""
|
||||
|
||||
long_prefill_token_threshold: int = 0
|
||||
long_prefill_token_threshold: int = Field(default=0, ge=0)
|
||||
"""For chunked prefill, a request is considered long if the prompt is
|
||||
longer than this number of tokens."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user