[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-09-21 09:56:11 -07:00
committed by GitHub
parent 4f14516790
commit 986e217158
30 changed files with 340 additions and 18 deletions
+3
View File
@@ -32,6 +32,7 @@ from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
from vllm.utils.async_utils import make_async
from vllm.utils.jsontree import json_iter_leaves
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.kv_hints import KvHintsEnvelope
logger = init_logger(__name__)
@@ -318,6 +319,7 @@ class InputProcessor:
data_parallel_rank: int | None = None,
resumable: bool = False,
session_id: str | None = None,
kv_hints: KvHintsEnvelope | None = None,
) -> EngineCoreRequest:
self._validate_params(params, supported_tasks)
self._validate_lora(lora_request)
@@ -461,6 +463,7 @@ class InputProcessor:
trace_headers=trace_headers,
resumable=resumable,
session_id=session_id,
kv_hints=kv_hints,
)
def _validate_prompt_len(