[Feat][RL] Pause and Resume with keep requests for single engine (#32351)
Signed-off-by: ahao-anyscale <ahao@anyscale.com> Signed-off-by: Aaron Hao <ahao@anyscale.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
import enum
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -18,6 +18,12 @@ from vllm.v1.metrics.stats import SchedulerStats
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
from vllm.v1.serial_utils import UtilityResult
|
||||
|
||||
# Type for pause_generation mode parameter.
|
||||
# - "abort": Abort all in-flight requests immediately (default).
|
||||
# - "wait": Wait for in-flight requests to complete before pausing.
|
||||
# - "keep": Freeze requests in queue; they resume on resume_generation().
|
||||
PauseMode = Literal["abort", "wait", "keep"]
|
||||
|
||||
# These are possible values of RequestOutput.finish_reason,
|
||||
# so form part of the external API.
|
||||
FINISH_REASON_STRINGS = ("stop", "length", "abort", "error")
|
||||
|
||||
Reference in New Issue
Block a user