[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -32,6 +32,7 @@ from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.jsontree import json_iter_leaves
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -318,6 +319,7 @@ class InputProcessor:
|
||||
data_parallel_rank: int | None = None,
|
||||
resumable: bool = False,
|
||||
session_id: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> EngineCoreRequest:
|
||||
self._validate_params(params, supported_tasks)
|
||||
self._validate_lora(lora_request)
|
||||
@@ -461,6 +463,7 @@ class InputProcessor:
|
||||
trace_headers=trace_headers,
|
||||
resumable=resumable,
|
||||
session_id=session_id,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
def _validate_prompt_len(
|
||||
|
||||
Reference in New Issue
Block a user