[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-09-21 09:56:11 -07:00
committed by GitHub
parent 4f14516790
commit 986e217158
30 changed files with 340 additions and 18 deletions
+2
View File
@@ -22,6 +22,7 @@ from vllm.tasks import SupportedTask
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.input_processor import InputProcessor
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
from vllm.v1.kv_hints import KvHintsEnvelope
if TYPE_CHECKING:
from vllm.v1.engine import PauseMode
@@ -102,6 +103,7 @@ class EngineClient(ABC):
priority: int = 0,
data_parallel_rank: int | None = None,
session_id: str | None = None,
kv_hints: KvHintsEnvelope | None = None,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[RequestOutput, None]: