diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 4eaf6f92f4..2bc35bcb64 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -56,6 +56,21 @@ message GenerateRequest { // Apply the engine's configured watermark. Defaults to true when omitted. optional bool watermarking = 16; + + KVHintsEnvelope kv_hints = 17; +} + +message KVHintsEnvelope { + string protocol_version = 1; + string message_id = 2; + repeated KVHintAction actions = 3; +} + +message KVHintAction { + string action_id = 1; + string action_type = 2; + string action_version = 3; + google.protobuf.Struct payload = 4; } message RandomSampling { diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 0273396ebd..d1a34455d2 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -194,6 +194,7 @@ impl ChatRequestProcessor { add_special_tokens: request.add_special_tokens, data_parallel_rank: request.data_parallel_rank, session_id: request.session_id, + kv_hints: None, reasoning_parser_kwargs, lora_request: request.lora_request, arrival_time: Some(arrival_time), diff --git a/rust/src/engine-core-client/src/protocol/kv_hints.rs b/rust/src/engine-core-client/src/protocol/kv_hints.rs new file mode 100644 index 0000000000..418ca2fced --- /dev/null +++ b/rust/src/engine-core-client/src/protocol/kv_hints.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvHintAction { + pub action_id: String, + pub action_type: String, + pub action_version: String, + pub payload: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvHintsEnvelope { + pub protocol_version: String, + pub message_id: String, + pub actions: Vec, +} diff --git a/rust/src/engine-core-client/src/protocol/mod.rs b/rust/src/engine-core-client/src/protocol/mod.rs index 742fed59bd..0586dde135 100644 --- a/rust/src/engine-core-client/src/protocol/mod.rs +++ b/rust/src/engine-core-client/src/protocol/mod.rs @@ -24,6 +24,7 @@ fn is_false(v: &bool) -> bool { pub mod dtype; pub mod handshake; +pub mod kv_hints; pub mod logprobs; pub mod lora; pub mod multimodal; diff --git a/rust/src/engine-core-client/src/protocol/request.rs b/rust/src/engine-core-client/src/protocol/request.rs index 973ada71c6..58c6efb641 100644 --- a/rust/src/engine-core-client/src/protocol/request.rs +++ b/rust/src/engine-core-client/src/protocol/request.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use serde_default::DefaultFromSerde; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; +use crate::protocol::kv_hints::KvHintsEnvelope; use crate::protocol::multimodal::MmFeatures; use crate::protocol::sampling::EngineCoreSamplingParams; use crate::protocol::{OpaqueValue, lora}; @@ -127,6 +128,8 @@ pub struct EngineCoreRequest { /// Stable session identity shared by related requests. #[serde(default)] pub session_id: Option, + #[serde(default)] + pub kv_hints: Option, } impl EngineCoreRequest { @@ -159,6 +162,7 @@ mod tests { use super::*; use crate::protocol::dtype::TensorDtype; + use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope}; use crate::protocol::multimodal::{ MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, MmModality, PlaceholderRange, @@ -181,6 +185,16 @@ mod tests { arrival_time: 1234.5, client_index: 7, session_id: Some("session-1".to_string()), + kv_hints: Some(KvHintsEnvelope { + protocol_version: "0.1".to_string(), + message_id: "msg-1".to_string(), + actions: vec![KvHintAction { + action_id: "action-1".to_string(), + action_type: "example.action".to_string(), + action_version: "1.0".to_string(), + payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]), + }], + }), ..EngineCoreRequest::default() }; @@ -191,13 +205,14 @@ mod tests { other => panic!("expected array, got {other:?}"), }; - assert_eq!(array.len(), 21); + assert_eq!(array.len(), 22); assert_eq!(array[0], Value::from("req-1")); assert_eq!(array[2], Value::Nil); assert_eq!(array[4], Value::Nil); assert_eq!(array[10], Value::Nil); assert_eq!(array[11], Value::from(7)); assert_eq!(array[20], Value::from("session-1")); + assert!(matches!(&array[21], Value::Map(_))); } #[test] diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index eb8e713fb4..bd3847c483 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -20,6 +20,7 @@ use zeromq::util::PeerIdentity; use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, ZmqMessage}; use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage}; +use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope}; use crate::protocol::logprobs::MaybeWireLogprobs; use crate::protocol::multimodal::{ MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice, @@ -165,6 +166,16 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest { }), arrival_time: 42.5, session_id: Some("session-1".to_string()), + kv_hints: Some(KvHintsEnvelope { + protocol_version: "0.1".to_string(), + message_id: "msg-1".to_string(), + actions: vec![KvHintAction { + action_id: "action-1".to_string(), + action_type: "example.action".to_string(), + action_version: "1.0".to_string(), + payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]), + }], + }), ..EngineCoreRequest::default() } } diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index a19dc9fdaa..76978328bb 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -52,6 +52,19 @@ class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True): output_kind: RequestOutputKind = RequestOutputKind.DELTA +class KvHintAction(msgspec.Struct, frozen=True): + action_id: str + action_type: str + action_version: str + payload: dict[str, object] + + +class KvHintsEnvelope(msgspec.Struct, frozen=True): + protocol_version: str + message_id: str + actions: list[KvHintAction] + + class EngineCoreRequest( msgspec.Struct, array_like=True, @@ -78,6 +91,7 @@ class EngineCoreRequest( reasoning_parser_kwargs: dict[str, object] | None = None abort_immediately: bool = False session_id: str | None = None + kv_hints: KvHintsEnvelope | None = None class EngineCoreOutput( @@ -153,6 +167,18 @@ request = EngineCoreRequest( arrival_time=42.5, client_index=0, session_id="session-1", + kv_hints=KvHintsEnvelope( + protocol_version="0.1", + message_id="msg-1", + actions=[ + KvHintAction( + action_id="action-1", + action_type="example.action", + action_version="1.0", + payload={"key": "value"}, + ) + ], + ), ) # All defaults -> empty map. Regression guard for the sparse-map decode. diff --git a/rust/src/llm/examples/external_engine_smoke.rs b/rust/src/llm/examples/external_engine_smoke.rs index 9d318285ae..0a580a2221 100644 --- a/rust/src/llm/examples/external_engine_smoke.rs +++ b/rust/src/llm/examples/external_engine_smoke.rs @@ -60,6 +60,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest { priority: 0, data_parallel_rank: None, session_id: None, + kv_hints: None, reasoning_parser_kwargs: None, lora_request: None, } diff --git a/rust/src/llm/src/request.rs b/rust/src/llm/src/request.rs index c647f54d8d..1f9d38ae0e 100644 --- a/rust/src/llm/src/request.rs +++ b/rust/src/llm/src/request.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use uuid::Uuid; +use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope; use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::multimodal::MmFeatures; use vllm_engine_core_client::protocol::request::{EngineCoreRequest, ReasoningParserKwargs}; @@ -48,6 +49,8 @@ pub struct GenerateRequest { pub data_parallel_rank: Option, /// Stable session identity shared by related requests. pub session_id: Option, + /// Optional orchestrator-originated KV hints. + pub kv_hints: Option, /// Optional reasoning-parser kwargs forwarded to engine-side structured /// output logic. pub reasoning_parser_kwargs: Option, @@ -79,6 +82,7 @@ impl GenerateRequest { priority, data_parallel_rank, session_id, + kv_hints, reasoning_parser_kwargs, lora_request, } = self; @@ -109,6 +113,7 @@ impl GenerateRequest { trace_headers, resumable: false, session_id, + kv_hints, external_req_id: Some(external_request_id), // Rust parser doesn't expose this information, leave it unset and let the // reasoning logic in engine-sided structured output manager handle it. @@ -134,6 +139,7 @@ impl PreparedGenerateRequest { mod tests { use std::collections::BTreeMap; + use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope}; use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; @@ -155,6 +161,16 @@ mod tests { priority: 3, data_parallel_rank: Some(2), session_id: Some("session-1".to_string()), + kv_hints: Some(KvHintsEnvelope { + protocol_version: "0.1".to_string(), + message_id: "msg-1".to_string(), + actions: vec![KvHintAction { + action_id: "action-1".to_string(), + action_type: "example.action".to_string(), + action_version: "1.0".to_string(), + payload: BTreeMap::new(), + }], + }), reasoning_parser_kwargs: Some(ReasoningParserKwargs { chat_template_kwargs: [( "chat_template_kwargs".to_string(), @@ -183,6 +199,10 @@ mod tests { assert_eq!(request.cache_salt.as_deref(), Some("salt")); assert_eq!(request.data_parallel_rank, Some(2)); assert_eq!(request.session_id.as_deref(), Some("session-1")); + assert_eq!( + request.kv_hints.as_ref().map(|hints| hints.message_id.as_str()), + Some("msg-1") + ); assert_eq!( request.trace_headers, Some(BTreeMap::from([( diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index c341737a3c..c9849cd57c 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -167,6 +167,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest priority: 0, data_parallel_rank: None, session_id: None, + kv_hints: None, reasoning_parser_kwargs: None, lora_request: None, } diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 0f28e147ef..538167b41f 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -7,6 +7,7 @@ use thiserror_ext::AsReport as _; use tonic::Status; use uuid::Uuid; +use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope}; use vllm_engine_core_client::protocol::output::StopReason; use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; use vllm_text::{ @@ -16,6 +17,32 @@ use vllm_text::{ use super::pb; +fn kv_hints_from_proto(hints: pb::KvHintsEnvelope) -> KvHintsEnvelope { + KvHintsEnvelope { + protocol_version: hints.protocol_version, + message_id: hints.message_id, + actions: hints + .actions + .into_iter() + .map(|action| KvHintAction { + action_id: action.action_id, + action_type: action.action_type, + action_version: action.action_version, + payload: action + .payload + .map(|payload| { + payload + .fields + .into_iter() + .map(|(key, value)| (key, proto_value_to_json(&value))) + .collect() + }) + .unwrap_or_default(), + }) + .collect(), + } +} + // ======================================================================================== // Request conversion // ======================================================================================== @@ -56,6 +83,7 @@ pub fn to_text_request( req.request_id }; let session_id = req.session_id.filter(|s| !s.is_empty()); + let kv_hints = req.kv_hints.map(kv_hints_from_proto); let sampling = req.sampling.as_ref(); let decoding = req.decoding.as_ref(); @@ -108,6 +136,7 @@ pub fn to_text_request( add_special_tokens: true, data_parallel_rank: None, session_id, + kv_hints, reasoning_parser_kwargs: None, lora_request: None, arrival_time: None, @@ -537,7 +566,9 @@ mod tests { use vllm_tokenizer::test_utils::TestTokenizer; use super::pb::finish_info::{FinishReason as PbFinishReason, StopReason as PbStopReason}; - use super::{ResponseOpts, pb, to_finish_info, to_sequence_output, to_text_request}; + use super::{ + ResponseOpts, json_to_proto_struct, pb, to_finish_info, to_sequence_output, to_text_request, + }; fn base_request() -> pb::GenerateRequest { pb::GenerateRequest { @@ -578,6 +609,35 @@ mod tests { assert_eq!(text.sampling_params.temperature, Some(0.7)); } + #[test] + fn kv_hints_propagate_from_top_level_request_field() { + let req = pb::GenerateRequest { + kv_hints: Some(pb::KvHintsEnvelope { + protocol_version: "0.1".to_string(), + message_id: "msg-1".to_string(), + actions: vec![pb::KvHintAction { + action_id: "action-1".to_string(), + action_type: "example.action".to_string(), + action_version: "1.0".to_string(), + payload: json_to_proto_struct(&serde_json::json!({"source": "g2"})), + }], + }), + ..base_request() + }; + + let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok"); + let hints = text.kv_hints.expect("kv hints"); + assert_eq!(hints.protocol_version, "0.1"); + assert_eq!(hints.message_id, "msg-1"); + assert_eq!(hints.actions[0].action_id, "action-1"); + assert_eq!(hints.actions[0].action_type, "example.action"); + assert_eq!(hints.actions[0].action_version, "1.0"); + assert_eq!( + hints.actions[0].payload.get("source"), + Some(&serde_json::json!("g2")) + ); + } + #[test] fn unset_temperature_defaults_to_greedy() { let text = to_text_request(base_request(), false, &["test-model".to_string()]) diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index 8093a34ca2..c3ec49dd23 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -82,6 +82,7 @@ pub(super) fn prepare_generate_request( add_special_tokens: false, data_parallel_rank: ctx.data_parallel_rank, session_id: ctx.session_id, + kv_hints: None, reasoning_parser_kwargs: None, lora_request: lora_resolution.lora_request.clone(), arrival_time: None, diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index a4e8b6d3e4..ddace3970e 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -155,6 +155,7 @@ pub(super) fn prepare_completion_request( add_special_tokens: request.add_special_tokens, data_parallel_rank: ctx.data_parallel_rank, session_id, + kv_hints: None, reasoning_parser_kwargs: None, lora_request: lora_resolution.lora_request.clone(), arrival_time: None, diff --git a/rust/src/server/src/routes/tokenize/types.rs b/rust/src/server/src/routes/tokenize/types.rs index 35a027c1c0..ba27211ebc 100644 --- a/rust/src/server/src/routes/tokenize/types.rs +++ b/rust/src/server/src/routes/tokenize/types.rs @@ -54,6 +54,7 @@ impl TokenizeCompletionRequest { add_special_tokens: self.add_special_tokens, data_parallel_rank: None, session_id: None, + kv_hints: None, reasoning_parser_kwargs: None, lora_request: None, arrival_time: None, diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 466b9d5b31..d8c6399773 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -60,6 +60,7 @@ pub fn lower_text_request( priority: request.priority, data_parallel_rank: request.data_parallel_rank, session_id: request.session_id.clone(), + kv_hints: request.kv_hints.clone(), reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(), lora_request: request.lora_request.clone(), arrival_time: request.arrival_time, @@ -320,6 +321,7 @@ mod tests { use std::collections::{BTreeSet, HashMap}; use serial_test::file_serial; + use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope}; use vllm_engine_core_client::protocol::multimodal::{ MmFeatureSpec, MmModality, PlaceholderRange, }; @@ -1301,6 +1303,35 @@ mod tests { assert_eq!(prepared.generate_request.request_id, "text-1"); } + #[test] + fn lower_text_request_passes_kv_hints_through() { + let hints = KvHintsEnvelope { + protocol_version: "0.1".to_string(), + message_id: "msg-1".to_string(), + actions: vec![KvHintAction { + action_id: "action-1".to_string(), + action_type: "example.action".to_string(), + action_version: "1.0".to_string(), + payload: Default::default(), + }], + }; + let request = TextRequest { + kv_hints: Some(hints.clone()), + ..sample_request() + }; + + let prepared = lower_text_request( + request, + vec![1, 2, 3], + sample_sampling_hints(), + sample_sampling_limits(), + &stub_tokenizer(), + ) + .unwrap(); + + assert_eq!(prepared.generate_request.kv_hints, Some(hints)); + } + #[test] fn lower_text_request_passes_arrival_time_through() { let request = TextRequest { diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index f0dd2fb1f8..7890977ae4 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use enum_as_inner::EnumAsInner; use serde::{Deserialize, Serialize}; use serde_json::Value; +use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope; use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::multimodal::MmFeatures; use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; @@ -211,6 +212,9 @@ pub struct TextRequest { /// Stable session identity shared by related requests. #[serde(default)] pub session_id: Option, + /// Optional orchestrator-originated KV hints. + #[serde(default)] + pub kv_hints: Option, /// Optional reasoning-parser kwargs forwarded to engine-side structured /// output logic. #[serde(default)] @@ -242,6 +246,7 @@ impl TextRequest { add_special_tokens: false, data_parallel_rank: None, session_id: None, + kv_hints: None, reasoning_parser_kwargs: None, lora_request: None, arrival_time: None, diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 035fb6b005..a691295616 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -33,6 +33,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( OffloadingConnectorScheduler, RequestOffloadState, + _create_req_context, get_sliding_window_size_in_chunks, ) from vllm.v1.core.block_pool import BlockPool @@ -60,6 +61,7 @@ from vllm.v1.kv_cache_interface import ( SlidingWindowSpec, ) from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry +from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.kv_offload.base import ( GPULoadStoreSpec, LookupResult, @@ -214,6 +216,7 @@ def _make_partial_tail_request( request = MagicMock() request.request_id = "req" request.kv_transfer_params = kv_transfer_params + request.kv_hints = None request.num_prompt_tokens = 30 request.num_tokens = 30 request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)] @@ -225,6 +228,21 @@ def _make_partial_tail_request( return request +def test_create_req_context_preserves_kv_hints(): + kv_hints = KvHintsEnvelope( + protocol_version="0.1", + message_id="message-1", + actions=[], + ) + request = MagicMock( + request_id="req", + kv_transfer_params=None, + kv_hints=kv_hints, + ) + + assert _create_req_context(request).kv_hints is kv_hints + + def _reduce_kv_connector_stats(runner): reduced: dict[str, int | float] = {} for payload in runner.kv_connector_stats: diff --git a/tests/v1/kv_offload/tiering/test_kvcr_tier.py b/tests/v1/kv_offload/tiering/test_kvcr_tier.py index 38a2ae044c..5cbc85230f 100644 --- a/tests/v1/kv_offload/tiering/test_kvcr_tier.py +++ b/tests/v1/kv_offload/tiering/test_kvcr_tier.py @@ -5,12 +5,13 @@ from collections.abc import Collection, Iterable, Mapping from types import SimpleNamespace from typing import Any +import msgspec import numpy as np import pytest pytest.importorskip("kvcr") -from kvcr import ROUTER_HINT_KEY, KVCRBindings +from kvcr import KVCRBindings from kvcr.config import G3Options, KVCRBackendConfigs, KVCRConfig, KVCRGuardConfig from kvcr.policy import FIFOPolicy, G3FIFOPolicy, G3LRUPolicy, LRUPolicy from kvcr.types import ( @@ -27,6 +28,7 @@ from kvcr.types import ( from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, ) +from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.kv_offload.base import ( LookupResult, Medium, @@ -271,10 +273,10 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch): """Check hint forwarding, key conversion, load descriptors, and cleanup.""" kvcr = RecordingKVCR() tier = _make_tier(monkeypatch, kvcr) - router_hint = {"opaque": True} + kv_hint = KvHintsEnvelope(protocol_version="0.1", message_id="msg", actions=[]) ctx = ReqContext( req_id="req", - kv_transfer_params={ROUTER_HINT_KEY: router_hint, "unrelated": object()}, + kv_hints=kv_hint, ) key = make_offload_key((123).to_bytes(8, "big"), 0) same_hash_other_group = make_offload_key((123).to_bytes(8, "big"), 7) @@ -282,7 +284,9 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch): tier.on_new_request(ctx) - assert kvcr.submit_hint_calls == [((), {"request_id": "req", "hints": router_hint})] + assert kvcr.submit_hint_calls == [ + ((), {"request_id": "req", "hints": msgspec.to_builtins(kv_hint)}) + ] bindings = kvcr.constructor_bindings assert bindings is not None diff --git a/tests/v1/test_request.py b/tests/v1/test_request.py index be417b9b2f..6c45e1a6d0 100644 --- a/tests/v1/test_request.py +++ b/tests/v1/test_request.py @@ -3,6 +3,7 @@ from vllm import SamplingParams from vllm.v1.engine import EngineCoreRequest +from vllm.v1.kv_hints import KvHintAction, KvHintsEnvelope from vllm.v1.request import Request, RequestStatus @@ -24,6 +25,7 @@ def test_request_status_fmt_str(): def test_request_copies_session_id_from_engine_core_request(): + """Test that Request preserves session ID from EngineCoreRequest.""" engine_request = EngineCoreRequest( request_id="request-1", prompt_token_ids=[1, 2, 3], @@ -40,3 +42,35 @@ def test_request_copies_session_id_from_engine_core_request(): request = Request.from_engine_core_request(engine_request, block_hasher=None) assert request.session_id == "session-1" + + +def test_request_copies_kv_hints_from_engine_core_request(): + """Test that Request preserves KV hints from EngineCoreRequest.""" + kv_hints = KvHintsEnvelope( + protocol_version="0.1", + message_id="msg-1", + actions=[ + KvHintAction( + action_id="action-1", + action_type="example.action", + action_version="1.0", + payload={}, + ) + ], + ) + engine_request = EngineCoreRequest( + request_id="request-1", + prompt_token_ids=[1, 2, 3], + mm_features=None, + sampling_params=SamplingParams(max_tokens=1), + pooling_params=None, + arrival_time=0.0, + lora_request=None, + cache_salt=None, + data_parallel_rank=None, + kv_hints=kv_hints, + ) + + request = Request.from_engine_core_request(engine_request, block_hasher=None) + + assert request.kv_hints == kv_hints diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index d6c210a6f9..47aa8654fe 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -553,6 +553,7 @@ def _create_req_context(req: Request) -> ReqContext: return ReqContext( req_id=req.request_id, kv_transfer_params=params, + kv_hints=req.kv_hints, load_tier_filter=load_filter, ) diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 1e6011a230..74849a85ce 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -22,6 +22,7 @@ from vllm.tasks import SupportedTask from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.input_processor import InputProcessor from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult +from vllm.v1.kv_hints import KvHintsEnvelope if TYPE_CHECKING: from vllm.v1.engine import PauseMode @@ -102,6 +103,7 @@ class EngineClient(ABC): priority: int = 0, data_parallel_rank: int | None = None, session_id: str | None = None, + kv_hints: KvHintsEnvelope | None = None, reasoning_ended: bool | None = None, reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> AsyncGenerator[RequestOutput, None]: diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index 7c424ea0df..e5bbb96a9c 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -16,6 +16,7 @@ from vllm.lora.request import LoRARequest from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams +from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.metrics.stats import ( PrefillStats, RequestSpecDecodeMetrics, @@ -157,6 +158,7 @@ class EngineCoreRequest( abort_immediately: bool = False session_id: str | None = None + kv_hints: KvHintsEnvelope | None = None @property def params(self) -> SamplingParams | PoolingParams: diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index ee913b052f..7d71761990 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -53,6 +53,7 @@ from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollec from vllm.v1.engine.parallel_sampling import ParentRequest from vllm.v1.executor import Executor from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult +from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.metrics.loggers import ( StatLoggerFactory, StatLoggerManager, @@ -386,6 +387,7 @@ class AsyncLLM(EngineClient): prompt_text: str | None = None, reasoning_ended: bool | None = None, reasoning_parser_kwargs: dict[str, Any] | None = None, + kv_hints: KvHintsEnvelope | None = None, ) -> RequestOutputCollector: """Add new request to the AsyncLLM.""" if self.errored: @@ -420,6 +422,7 @@ class AsyncLLM(EngineClient): priority, data_parallel_rank, session_id, + kv_hints, ) # Convert Input --> Request. @@ -453,6 +456,7 @@ class AsyncLLM(EngineClient): priority=priority, data_parallel_rank=data_parallel_rank, session_id=session_id, + kv_hints=kv_hints, ) else: # Raw prompts require tokenization and possibly multimodal @@ -469,6 +473,7 @@ class AsyncLLM(EngineClient): priority=priority, data_parallel_rank=data_parallel_rank, session_id=session_id, + kv_hints=kv_hints, ) prompt_text, _, _ = extract_prompt_components(self.model_config, prompt) @@ -557,6 +562,7 @@ class AsyncLLM(EngineClient): priority: int = 0, data_parallel_rank: int | None = None, session_id: str | None = None, + kv_hints: KvHintsEnvelope | None = None, ) -> RequestOutputCollector: self._validate_streaming_input_sampling_params(sampling_params) @@ -569,6 +575,7 @@ class AsyncLLM(EngineClient): priority=priority, data_parallel_rank=data_parallel_rank, session_id=session_id, + kv_hints=kv_hints, ) if not sampling_params.skip_clone: @@ -670,6 +677,7 @@ class AsyncLLM(EngineClient): priority: int = 0, data_parallel_rank: int | None = None, session_id: str | None = None, + kv_hints: KvHintsEnvelope | None = None, reasoning_ended: bool | None = None, reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> AsyncGenerator[RequestOutput, None]: @@ -710,6 +718,7 @@ class AsyncLLM(EngineClient): priority=priority, data_parallel_rank=data_parallel_rank, session_id=session_id, + kv_hints=kv_hints, prompt_text=prompt_text, reasoning_ended=reasoning_ended, reasoning_parser_kwargs=reasoning_parser_kwargs, diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index d0ccb1dacf..582e1abac6 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -32,6 +32,7 @@ from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid from vllm.utils.async_utils import make_async from vllm.utils.jsontree import json_iter_leaves from vllm.v1.engine import EngineCoreRequest +from vllm.v1.kv_hints import KvHintsEnvelope logger = init_logger(__name__) @@ -318,6 +319,7 @@ class InputProcessor: data_parallel_rank: int | None = None, resumable: bool = False, session_id: str | None = None, + kv_hints: KvHintsEnvelope | None = None, ) -> EngineCoreRequest: self._validate_params(params, supported_tasks) self._validate_lora(lora_request) @@ -461,6 +463,7 @@ class InputProcessor: trace_headers=trace_headers, resumable=resumable, session_id=session_id, + kv_hints=kv_hints, ) def _validate_prompt_len( diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index a2d97b5544..c925ba90e8 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -34,6 +34,7 @@ from vllm.v1.engine.input_processor import InputProcessor from vllm.v1.engine.output_processor import OutputProcessor from vllm.v1.engine.parallel_sampling import ParentRequest from vllm.v1.executor import Executor +from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.metrics.loggers import StatLoggerFactory, StatLoggerManager from vllm.v1.metrics.reader import Metric, get_metrics_snapshot from vllm.v1.metrics.stats import IterationStats @@ -230,6 +231,7 @@ class LLMEngine: priority: int = 0, session_id: str | None = None, prompt_text: str | None = None, + kv_hints: KvHintsEnvelope | None = None, ) -> str: # Validate the request_id type. if not isinstance(request_id, str): @@ -263,6 +265,7 @@ class LLMEngine: trace_headers=trace_headers, priority=priority, session_id=session_id, + kv_hints=kv_hints, ) prompt_text, _, _ = extract_prompt_components(self.model_config, prompt) diff --git a/vllm/v1/kv_hints/__init__.py b/vllm/v1/kv_hints/__init__.py new file mode 100644 index 0000000000..d0807572c7 --- /dev/null +++ b/vllm/v1/kv_hints/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.v1.kv_hints.protocol import KvHintAction, KvHintsEnvelope + +__all__ = ["KvHintAction", "KvHintsEnvelope"] diff --git a/vllm/v1/kv_hints/protocol.py b/vllm/v1/kv_hints/protocol.py new file mode 100644 index 0000000000..a3f93ab0bf --- /dev/null +++ b/vllm/v1/kv_hints/protocol.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import msgspec + + +class KvHintAction(msgspec.Struct, frozen=True): # type: ignore[call-arg] + """A versioned orchestrator-provided KV hint action.""" + + action_id: str + action_type: str + action_version: str + # `payload` must contain JSON-compatible values. Action-specific schemas are + # validated by the component implementing the action. + payload: dict[str, Any] + + +class KvHintsEnvelope(msgspec.Struct, frozen=True): # type: ignore[call-arg] + """Orchestrator-provided KV hints optionally attached to one inference request. + A KV hint envelope is a collection of KV hint action(s).""" + + protocol_version: str + message_id: str + actions: list[KvHintAction] diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 1144477628..f8e92abed6 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: OffloadingConnectorStats, ) +from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.kv_offload.config import OffloadingConfig # `OffloadKey` identifies an offloaded block. It combines a block hash with @@ -89,10 +90,11 @@ TierFilter.ALL = TierFilter(matchers=(TierMatcher(),)) class ReqContext: req_id: str kv_transfer_params: dict[str, Any] | None = None + kv_hints: KvHintsEnvelope | None = None load_tier_filter: TierFilter = TierFilter.ALL # Per-request scratch space keyed by value type, so a tier can parse - # kv_transfer_params once (in on_new_request) and read the result back - # on later calls for the same request. + # kv_transfer_params and kv_hints once (in on_new_request) and read the + # result back on later calls for the same request. _state: dict[type, Any] = field(default_factory=dict, repr=False, init=False) # End-token position for each key in this request. The scheduler records # these positions so managers can recover prefix order even when store diff --git a/vllm/v1/kv_offload/tiering/kvcr/manager.py b/vllm/v1/kv_offload/tiering/kvcr/manager.py index 00186279a4..18da2a6c27 100644 --- a/vllm/v1/kv_offload/tiering/kvcr/manager.py +++ b/vllm/v1/kv_offload/tiering/kvcr/manager.py @@ -7,15 +7,15 @@ import mmap import socket import time import uuid -from collections.abc import Collection, Iterable, Mapping +from collections.abc import Collection, Iterable from pathlib import Path from typing import TYPE_CHECKING, Any, cast +import msgspec from kvcr import ( DURATION_METRIC, KVCR, ROUTER_HINT_CAPABILITIES, - ROUTER_HINT_KEY, STATE_METRIC, TRANSFER_BLOCKS_METRIC, TRANSFER_BYTES_METRIC, @@ -575,14 +575,11 @@ class KVCRSecondaryTierManager(SecondaryTierManager): @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: - params = getattr(req_context, "kv_transfer_params", None) - if isinstance(params, Mapping): - hint = params.get(ROUTER_HINT_KEY) - if hint is not None: - self._kvcr.submit_hint( - request_id=req_context.req_id, - hints=hint, - ) + if (hint := req_context.kv_hints) is not None: + self._kvcr.submit_hint( + request_id=req_context.req_id, + hints=msgspec.to_builtins(hint), + ) return RequestOffloadingContext() @override diff --git a/vllm/v1/request.py b/vllm/v1/request.py index dcf679093b..4b1b9ecc2e 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -20,6 +20,7 @@ from vllm.v1.engine import ( EngineCoreRequest, FinishReason, ) +from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.metrics.stats import PrefillStats, RequestSpecDecodeMetrics from vllm.v1.structured_output.request import StructuredOutputRequest from vllm.v1.utils import ConstantList @@ -78,6 +79,7 @@ class Request: reasoning_ended: bool | None = None, reasoning_parser_kwargs: dict[str, Any] | None = None, abort_immediately: bool = False, + kv_hints: KvHintsEnvelope | None = None, ) -> None: self.request_id = request_id self.client_index = client_index @@ -193,6 +195,7 @@ class Request: # trace_headers self.trace_headers = trace_headers self.session_id = session_id + self.kv_hints = kv_hints # True if this request is scheduled as a non-final prefill chunk. self.is_prefill_chunk = False @@ -260,6 +263,7 @@ class Request: block_hasher=block_hasher, resumable=request.resumable, session_id=request.session_id, + kv_hints=request.kv_hints, reasoning_ended=request.reasoning_ended, reasoning_parser_kwargs=request.reasoning_parser_kwargs, abort_immediately=request.abort_immediately,