[Scheduler] Add --max-num-active-seqs to cap RUNNING admission (#56758)
Signed-off-by: Li <chuali@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
b1f9ebc0c3
commit
f730a93d2b
@@ -556,6 +556,7 @@ class EngineArgs:
|
||||
max_num_scheduled_tokens: int | None = None
|
||||
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
|
||||
max_num_seqs: int | None = None
|
||||
max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs
|
||||
max_num_queued_reqs: int | None = None
|
||||
max_num_queued_tokens: int | None = None
|
||||
max_logprobs: int = ModelConfig.max_logprobs
|
||||
@@ -1589,6 +1590,10 @@ class EngineArgs:
|
||||
"default": None,
|
||||
},
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-active-seqs",
|
||||
**scheduler_kwargs["max_num_active_seqs"],
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
|
||||
)
|
||||
@@ -2421,6 +2426,7 @@ class EngineArgs:
|
||||
max_num_batched_tokens=self.max_num_batched_tokens,
|
||||
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
|
||||
max_num_seqs=self.max_num_seqs,
|
||||
max_num_active_seqs=self.max_num_active_seqs,
|
||||
max_num_queued_reqs=self.max_num_queued_reqs,
|
||||
max_num_queued_tokens=self.max_num_queued_tokens,
|
||||
max_model_len=model_config.max_model_len,
|
||||
|
||||
Reference in New Issue
Block a user