[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -56,6 +56,21 @@ message GenerateRequest {
|
||||
|
||||
// Apply the engine's configured watermark. Defaults to true when omitted.
|
||||
optional bool watermarking = 16;
|
||||
|
||||
KVHintsEnvelope kv_hints = 17;
|
||||
}
|
||||
|
||||
message KVHintsEnvelope {
|
||||
string protocol_version = 1;
|
||||
string message_id = 2;
|
||||
repeated KVHintAction actions = 3;
|
||||
}
|
||||
|
||||
message KVHintAction {
|
||||
string action_id = 1;
|
||||
string action_type = 2;
|
||||
string action_version = 3;
|
||||
google.protobuf.Struct payload = 4;
|
||||
}
|
||||
|
||||
message RandomSampling {
|
||||
|
||||
@@ -194,6 +194,7 @@ impl ChatRequestProcessor {
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
session_id: request.session_id,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request: request.lora_request,
|
||||
arrival_time: Some(arrival_time),
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct KvHintAction {
|
||||
pub action_id: String,
|
||||
pub action_type: String,
|
||||
pub action_version: String,
|
||||
pub payload: BTreeMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct KvHintsEnvelope {
|
||||
pub protocol_version: String,
|
||||
pub message_id: String,
|
||||
pub actions: Vec<KvHintAction>,
|
||||
}
|
||||
@@ -24,6 +24,7 @@ fn is_false(v: &bool) -> bool {
|
||||
|
||||
pub mod dtype;
|
||||
pub mod handshake;
|
||||
pub mod kv_hints;
|
||||
pub mod logprobs;
|
||||
pub mod lora;
|
||||
pub mod multimodal;
|
||||
|
||||
@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use crate::protocol::kv_hints::KvHintsEnvelope;
|
||||
use crate::protocol::multimodal::MmFeatures;
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::{OpaqueValue, lora};
|
||||
@@ -127,6 +128,8 @@ pub struct EngineCoreRequest {
|
||||
/// Stable session identity shared by related requests.
|
||||
#[serde(default)]
|
||||
pub session_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub kv_hints: Option<KvHintsEnvelope>,
|
||||
}
|
||||
|
||||
impl EngineCoreRequest {
|
||||
@@ -159,6 +162,7 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::dtype::TensorDtype;
|
||||
use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use crate::protocol::multimodal::{
|
||||
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, MmModality,
|
||||
PlaceholderRange,
|
||||
@@ -181,6 +185,16 @@ mod tests {
|
||||
arrival_time: 1234.5,
|
||||
client_index: 7,
|
||||
session_id: Some("session-1".to_string()),
|
||||
kv_hints: Some(KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]),
|
||||
}],
|
||||
}),
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
@@ -191,13 +205,14 @@ mod tests {
|
||||
other => panic!("expected array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array.len(), 21);
|
||||
assert_eq!(array.len(), 22);
|
||||
assert_eq!(array[0], Value::from("req-1"));
|
||||
assert_eq!(array[2], Value::Nil);
|
||||
assert_eq!(array[4], Value::Nil);
|
||||
assert_eq!(array[10], Value::Nil);
|
||||
assert_eq!(array[11], Value::from(7));
|
||||
assert_eq!(array[20], Value::from("session-1"));
|
||||
assert!(matches!(&array[21], Value::Map(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -20,6 +20,7 @@ use zeromq::util::PeerIdentity;
|
||||
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, ZmqMessage};
|
||||
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::multimodal::{
|
||||
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice,
|
||||
@@ -165,6 +166,16 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest {
|
||||
}),
|
||||
arrival_time: 42.5,
|
||||
session_id: Some("session-1".to_string()),
|
||||
kv_hints: Some(KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]),
|
||||
}],
|
||||
}),
|
||||
..EngineCoreRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,19 @@ class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
|
||||
output_kind: RequestOutputKind = RequestOutputKind.DELTA
|
||||
|
||||
|
||||
class KvHintAction(msgspec.Struct, frozen=True):
|
||||
action_id: str
|
||||
action_type: str
|
||||
action_version: str
|
||||
payload: dict[str, object]
|
||||
|
||||
|
||||
class KvHintsEnvelope(msgspec.Struct, frozen=True):
|
||||
protocol_version: str
|
||||
message_id: str
|
||||
actions: list[KvHintAction]
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
msgspec.Struct,
|
||||
array_like=True,
|
||||
@@ -78,6 +91,7 @@ class EngineCoreRequest(
|
||||
reasoning_parser_kwargs: dict[str, object] | None = None
|
||||
abort_immediately: bool = False
|
||||
session_id: str | None = None
|
||||
kv_hints: KvHintsEnvelope | None = None
|
||||
|
||||
|
||||
class EngineCoreOutput(
|
||||
@@ -153,6 +167,18 @@ request = EngineCoreRequest(
|
||||
arrival_time=42.5,
|
||||
client_index=0,
|
||||
session_id="session-1",
|
||||
kv_hints=KvHintsEnvelope(
|
||||
protocol_version="0.1",
|
||||
message_id="msg-1",
|
||||
actions=[
|
||||
KvHintAction(
|
||||
action_id="action-1",
|
||||
action_type="example.action",
|
||||
action_version="1.0",
|
||||
payload={"key": "value"},
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
# All defaults -> empty map. Regression guard for the sparse-map decode.
|
||||
|
||||
@@ -60,6 +60,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::request::{EngineCoreRequest, ReasoningParserKwargs};
|
||||
@@ -48,6 +49,8 @@ pub struct GenerateRequest {
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Stable session identity shared by related requests.
|
||||
pub session_id: Option<String>,
|
||||
/// Optional orchestrator-originated KV hints.
|
||||
pub kv_hints: Option<KvHintsEnvelope>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
@@ -79,6 +82,7 @@ impl GenerateRequest {
|
||||
priority,
|
||||
data_parallel_rank,
|
||||
session_id,
|
||||
kv_hints,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request,
|
||||
} = self;
|
||||
@@ -109,6 +113,7 @@ impl GenerateRequest {
|
||||
trace_headers,
|
||||
resumable: false,
|
||||
session_id,
|
||||
kv_hints,
|
||||
external_req_id: Some(external_request_id),
|
||||
// Rust parser doesn't expose this information, leave it unset and let the
|
||||
// reasoning logic in engine-sided structured output manager handle it.
|
||||
@@ -134,6 +139,7 @@ impl PreparedGenerateRequest {
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
||||
|
||||
@@ -155,6 +161,16 @@ mod tests {
|
||||
priority: 3,
|
||||
data_parallel_rank: Some(2),
|
||||
session_id: Some("session-1".to_string()),
|
||||
kv_hints: Some(KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: BTreeMap::new(),
|
||||
}],
|
||||
}),
|
||||
reasoning_parser_kwargs: Some(ReasoningParserKwargs {
|
||||
chat_template_kwargs: [(
|
||||
"chat_template_kwargs".to_string(),
|
||||
@@ -183,6 +199,10 @@ mod tests {
|
||||
assert_eq!(request.cache_salt.as_deref(), Some("salt"));
|
||||
assert_eq!(request.data_parallel_rank, Some(2));
|
||||
assert_eq!(request.session_id.as_deref(), Some("session-1"));
|
||||
assert_eq!(
|
||||
request.kv_hints.as_ref().map(|hints| hints.message_id.as_str()),
|
||||
Some("msg-1")
|
||||
);
|
||||
assert_eq!(
|
||||
request.trace_headers,
|
||||
Some(BTreeMap::from([(
|
||||
|
||||
@@ -167,6 +167,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tonic::Status;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use vllm_text::{
|
||||
@@ -16,6 +17,32 @@ use vllm_text::{
|
||||
|
||||
use super::pb;
|
||||
|
||||
fn kv_hints_from_proto(hints: pb::KvHintsEnvelope) -> KvHintsEnvelope {
|
||||
KvHintsEnvelope {
|
||||
protocol_version: hints.protocol_version,
|
||||
message_id: hints.message_id,
|
||||
actions: hints
|
||||
.actions
|
||||
.into_iter()
|
||||
.map(|action| KvHintAction {
|
||||
action_id: action.action_id,
|
||||
action_type: action.action_type,
|
||||
action_version: action.action_version,
|
||||
payload: action
|
||||
.payload
|
||||
.map(|payload| {
|
||||
payload
|
||||
.fields
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key, proto_value_to_json(&value)))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Request conversion
|
||||
// ========================================================================================
|
||||
@@ -56,6 +83,7 @@ pub fn to_text_request(
|
||||
req.request_id
|
||||
};
|
||||
let session_id = req.session_id.filter(|s| !s.is_empty());
|
||||
let kv_hints = req.kv_hints.map(kv_hints_from_proto);
|
||||
|
||||
let sampling = req.sampling.as_ref();
|
||||
let decoding = req.decoding.as_ref();
|
||||
@@ -108,6 +136,7 @@ pub fn to_text_request(
|
||||
add_special_tokens: true,
|
||||
data_parallel_rank: None,
|
||||
session_id,
|
||||
kv_hints,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
@@ -537,7 +566,9 @@ mod tests {
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::pb::finish_info::{FinishReason as PbFinishReason, StopReason as PbStopReason};
|
||||
use super::{ResponseOpts, pb, to_finish_info, to_sequence_output, to_text_request};
|
||||
use super::{
|
||||
ResponseOpts, json_to_proto_struct, pb, to_finish_info, to_sequence_output, to_text_request,
|
||||
};
|
||||
|
||||
fn base_request() -> pb::GenerateRequest {
|
||||
pb::GenerateRequest {
|
||||
@@ -578,6 +609,35 @@ mod tests {
|
||||
assert_eq!(text.sampling_params.temperature, Some(0.7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kv_hints_propagate_from_top_level_request_field() {
|
||||
let req = pb::GenerateRequest {
|
||||
kv_hints: Some(pb::KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![pb::KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: json_to_proto_struct(&serde_json::json!({"source": "g2"})),
|
||||
}],
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
|
||||
let hints = text.kv_hints.expect("kv hints");
|
||||
assert_eq!(hints.protocol_version, "0.1");
|
||||
assert_eq!(hints.message_id, "msg-1");
|
||||
assert_eq!(hints.actions[0].action_id, "action-1");
|
||||
assert_eq!(hints.actions[0].action_type, "example.action");
|
||||
assert_eq!(hints.actions[0].action_version, "1.0");
|
||||
assert_eq!(
|
||||
hints.actions[0].payload.get("source"),
|
||||
Some(&serde_json::json!("g2"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unset_temperature_defaults_to_greedy() {
|
||||
let text = to_text_request(base_request(), false, &["test-model".to_string()])
|
||||
|
||||
@@ -82,6 +82,7 @@ pub(super) fn prepare_generate_request(
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id: ctx.session_id,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
|
||||
@@ -155,6 +155,7 @@ pub(super) fn prepare_completion_request(
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
|
||||
@@ -54,6 +54,7 @@ impl TokenizeCompletionRequest {
|
||||
add_special_tokens: self.add_special_tokens,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
|
||||
@@ -60,6 +60,7 @@ pub fn lower_text_request(
|
||||
priority: request.priority,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
session_id: request.session_id.clone(),
|
||||
kv_hints: request.kv_hints.clone(),
|
||||
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
|
||||
lora_request: request.lora_request.clone(),
|
||||
arrival_time: request.arrival_time,
|
||||
@@ -320,6 +321,7 @@ mod tests {
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use serial_test::file_serial;
|
||||
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmFeatureSpec, MmModality, PlaceholderRange,
|
||||
};
|
||||
@@ -1301,6 +1303,35 @@ mod tests {
|
||||
assert_eq!(prepared.generate_request.request_id, "text-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_passes_kv_hints_through() {
|
||||
let hints = KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: Default::default(),
|
||||
}],
|
||||
};
|
||||
let request = TextRequest {
|
||||
kv_hints: Some(hints.clone()),
|
||||
..sample_request()
|
||||
};
|
||||
|
||||
let prepared = lower_text_request(
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(prepared.generate_request.kv_hints, Some(hints));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_passes_arrival_time_through() {
|
||||
let request = TextRequest {
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::collections::HashMap;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
@@ -211,6 +212,9 @@ pub struct TextRequest {
|
||||
/// Stable session identity shared by related requests.
|
||||
#[serde(default)]
|
||||
pub session_id: Option<String>,
|
||||
/// Optional orchestrator-originated KV hints.
|
||||
#[serde(default)]
|
||||
pub kv_hints: Option<KvHintsEnvelope>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
#[serde(default)]
|
||||
@@ -242,6 +246,7 @@ impl TextRequest {
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
OffloadingConnectorScheduler,
|
||||
RequestOffloadState,
|
||||
_create_req_context,
|
||||
get_sliding_window_size_in_chunks,
|
||||
)
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
@@ -60,6 +61,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
LookupResult,
|
||||
@@ -214,6 +216,7 @@ def _make_partial_tail_request(
|
||||
request = MagicMock()
|
||||
request.request_id = "req"
|
||||
request.kv_transfer_params = kv_transfer_params
|
||||
request.kv_hints = None
|
||||
request.num_prompt_tokens = 30
|
||||
request.num_tokens = 30
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
|
||||
@@ -225,6 +228,21 @@ def _make_partial_tail_request(
|
||||
return request
|
||||
|
||||
|
||||
def test_create_req_context_preserves_kv_hints():
|
||||
kv_hints = KvHintsEnvelope(
|
||||
protocol_version="0.1",
|
||||
message_id="message-1",
|
||||
actions=[],
|
||||
)
|
||||
request = MagicMock(
|
||||
request_id="req",
|
||||
kv_transfer_params=None,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
assert _create_req_context(request).kv_hints is kv_hints
|
||||
|
||||
|
||||
def _reduce_kv_connector_stats(runner):
|
||||
reduced: dict[str, int | float] = {}
|
||||
for payload in runner.kv_connector_stats:
|
||||
|
||||
@@ -5,12 +5,13 @@ from collections.abc import Collection, Iterable, Mapping
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("kvcr")
|
||||
|
||||
from kvcr import ROUTER_HINT_KEY, KVCRBindings
|
||||
from kvcr import KVCRBindings
|
||||
from kvcr.config import G3Options, KVCRBackendConfigs, KVCRConfig, KVCRGuardConfig
|
||||
from kvcr.policy import FIFOPolicy, G3FIFOPolicy, G3LRUPolicy, LRUPolicy
|
||||
from kvcr.types import (
|
||||
@@ -27,6 +28,7 @@ from kvcr.types import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
Medium,
|
||||
@@ -271,10 +273,10 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
|
||||
"""Check hint forwarding, key conversion, load descriptors, and cleanup."""
|
||||
kvcr = RecordingKVCR()
|
||||
tier = _make_tier(monkeypatch, kvcr)
|
||||
router_hint = {"opaque": True}
|
||||
kv_hint = KvHintsEnvelope(protocol_version="0.1", message_id="msg", actions=[])
|
||||
ctx = ReqContext(
|
||||
req_id="req",
|
||||
kv_transfer_params={ROUTER_HINT_KEY: router_hint, "unrelated": object()},
|
||||
kv_hints=kv_hint,
|
||||
)
|
||||
key = make_offload_key((123).to_bytes(8, "big"), 0)
|
||||
same_hash_other_group = make_offload_key((123).to_bytes(8, "big"), 7)
|
||||
@@ -282,7 +284,9 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
|
||||
|
||||
tier.on_new_request(ctx)
|
||||
|
||||
assert kvcr.submit_hint_calls == [((), {"request_id": "req", "hints": router_hint})]
|
||||
assert kvcr.submit_hint_calls == [
|
||||
((), {"request_id": "req", "hints": msgspec.to_builtins(kv_hint)})
|
||||
]
|
||||
|
||||
bindings = kvcr.constructor_bindings
|
||||
assert bindings is not None
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.kv_hints import KvHintAction, KvHintsEnvelope
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
|
||||
@@ -24,6 +25,7 @@ def test_request_status_fmt_str():
|
||||
|
||||
|
||||
def test_request_copies_session_id_from_engine_core_request():
|
||||
"""Test that Request preserves session ID from EngineCoreRequest."""
|
||||
engine_request = EngineCoreRequest(
|
||||
request_id="request-1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
@@ -40,3 +42,35 @@ def test_request_copies_session_id_from_engine_core_request():
|
||||
request = Request.from_engine_core_request(engine_request, block_hasher=None)
|
||||
|
||||
assert request.session_id == "session-1"
|
||||
|
||||
|
||||
def test_request_copies_kv_hints_from_engine_core_request():
|
||||
"""Test that Request preserves KV hints from EngineCoreRequest."""
|
||||
kv_hints = KvHintsEnvelope(
|
||||
protocol_version="0.1",
|
||||
message_id="msg-1",
|
||||
actions=[
|
||||
KvHintAction(
|
||||
action_id="action-1",
|
||||
action_type="example.action",
|
||||
action_version="1.0",
|
||||
payload={},
|
||||
)
|
||||
],
|
||||
)
|
||||
engine_request = EngineCoreRequest(
|
||||
request_id="request-1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
request = Request.from_engine_core_request(engine_request, block_hasher=None)
|
||||
|
||||
assert request.kv_hints == kv_hints
|
||||
|
||||
@@ -553,6 +553,7 @@ def _create_req_context(req: Request) -> ReqContext:
|
||||
return ReqContext(
|
||||
req_id=req.request_id,
|
||||
kv_transfer_params=params,
|
||||
kv_hints=req.kv_hints,
|
||||
load_tier_filter=load_filter,
|
||||
)
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.tasks import SupportedTask
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine import PauseMode
|
||||
@@ -102,6 +103,7 @@ class EngineClient(ABC):
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
session_id: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.metrics.stats import (
|
||||
PrefillStats,
|
||||
RequestSpecDecodeMetrics,
|
||||
@@ -157,6 +158,7 @@ class EngineCoreRequest(
|
||||
abort_immediately: bool = False
|
||||
|
||||
session_id: str | None = None
|
||||
kv_hints: KvHintsEnvelope | None = None
|
||||
|
||||
@property
|
||||
def params(self) -> SamplingParams | PoolingParams:
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollec
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.metrics.loggers import (
|
||||
StatLoggerFactory,
|
||||
StatLoggerManager,
|
||||
@@ -386,6 +387,7 @@ class AsyncLLM(EngineClient):
|
||||
prompt_text: str | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
"""Add new request to the AsyncLLM."""
|
||||
if self.errored:
|
||||
@@ -420,6 +422,7 @@ class AsyncLLM(EngineClient):
|
||||
priority,
|
||||
data_parallel_rank,
|
||||
session_id,
|
||||
kv_hints,
|
||||
)
|
||||
|
||||
# Convert Input --> Request.
|
||||
@@ -453,6 +456,7 @@ class AsyncLLM(EngineClient):
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
else:
|
||||
# Raw prompts require tokenization and possibly multimodal
|
||||
@@ -469,6 +473,7 @@ class AsyncLLM(EngineClient):
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
@@ -557,6 +562,7 @@ class AsyncLLM(EngineClient):
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
session_id: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
self._validate_streaming_input_sampling_params(sampling_params)
|
||||
|
||||
@@ -569,6 +575,7 @@ class AsyncLLM(EngineClient):
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
if not sampling_params.skip_clone:
|
||||
@@ -670,6 +677,7 @@ class AsyncLLM(EngineClient):
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
session_id: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
@@ -710,6 +718,7 @@ class AsyncLLM(EngineClient):
|
||||
priority=priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
kv_hints=kv_hints,
|
||||
prompt_text=prompt_text,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.jsontree import json_iter_leaves
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -318,6 +319,7 @@ class InputProcessor:
|
||||
data_parallel_rank: int | None = None,
|
||||
resumable: bool = False,
|
||||
session_id: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> EngineCoreRequest:
|
||||
self._validate_params(params, supported_tasks)
|
||||
self._validate_lora(lora_request)
|
||||
@@ -461,6 +463,7 @@ class InputProcessor:
|
||||
trace_headers=trace_headers,
|
||||
resumable=resumable,
|
||||
session_id=session_id,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
def _validate_prompt_len(
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.metrics.loggers import StatLoggerFactory, StatLoggerManager
|
||||
from vllm.v1.metrics.reader import Metric, get_metrics_snapshot
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
@@ -230,6 +231,7 @@ class LLMEngine:
|
||||
priority: int = 0,
|
||||
session_id: str | None = None,
|
||||
prompt_text: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> str:
|
||||
# Validate the request_id type.
|
||||
if not isinstance(request_id, str):
|
||||
@@ -263,6 +265,7 @@ class LLMEngine:
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
session_id=session_id,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.v1.kv_hints.protocol import KvHintAction, KvHintsEnvelope
|
||||
|
||||
__all__ = ["KvHintAction", "KvHintsEnvelope"]
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class KvHintAction(msgspec.Struct, frozen=True): # type: ignore[call-arg]
|
||||
"""A versioned orchestrator-provided KV hint action."""
|
||||
|
||||
action_id: str
|
||||
action_type: str
|
||||
action_version: str
|
||||
# `payload` must contain JSON-compatible values. Action-specific schemas are
|
||||
# validated by the component implementing the action.
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
class KvHintsEnvelope(msgspec.Struct, frozen=True): # type: ignore[call-arg]
|
||||
"""Orchestrator-provided KV hints optionally attached to one inference request.
|
||||
A KV hint envelope is a collection of KV hint action(s)."""
|
||||
|
||||
protocol_version: str
|
||||
message_id: str
|
||||
actions: list[KvHintAction]
|
||||
@@ -16,6 +16,7 @@ if TYPE_CHECKING:
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
|
||||
# `OffloadKey` identifies an offloaded block. It combines a block hash with
|
||||
@@ -89,10 +90,11 @@ TierFilter.ALL = TierFilter(matchers=(TierMatcher(),))
|
||||
class ReqContext:
|
||||
req_id: str
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
kv_hints: KvHintsEnvelope | None = None
|
||||
load_tier_filter: TierFilter = TierFilter.ALL
|
||||
# Per-request scratch space keyed by value type, so a tier can parse
|
||||
# kv_transfer_params once (in on_new_request) and read the result back
|
||||
# on later calls for the same request.
|
||||
# kv_transfer_params and kv_hints once (in on_new_request) and read the
|
||||
# result back on later calls for the same request.
|
||||
_state: dict[type, Any] = field(default_factory=dict, repr=False, init=False)
|
||||
# End-token position for each key in this request. The scheduler records
|
||||
# these positions so managers can recover prefix order even when store
|
||||
|
||||
@@ -7,15 +7,15 @@ import mmap
|
||||
import socket
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Collection, Iterable, Mapping
|
||||
from collections.abc import Collection, Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import msgspec
|
||||
from kvcr import (
|
||||
DURATION_METRIC,
|
||||
KVCR,
|
||||
ROUTER_HINT_CAPABILITIES,
|
||||
ROUTER_HINT_KEY,
|
||||
STATE_METRIC,
|
||||
TRANSFER_BLOCKS_METRIC,
|
||||
TRANSFER_BYTES_METRIC,
|
||||
@@ -575,14 +575,11 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
params = getattr(req_context, "kv_transfer_params", None)
|
||||
if isinstance(params, Mapping):
|
||||
hint = params.get(ROUTER_HINT_KEY)
|
||||
if hint is not None:
|
||||
self._kvcr.submit_hint(
|
||||
request_id=req_context.req_id,
|
||||
hints=hint,
|
||||
)
|
||||
if (hint := req_context.kv_hints) is not None:
|
||||
self._kvcr.submit_hint(
|
||||
request_id=req_context.req_id,
|
||||
hints=msgspec.to_builtins(hint),
|
||||
)
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.v1.engine import (
|
||||
EngineCoreRequest,
|
||||
FinishReason,
|
||||
)
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.metrics.stats import PrefillStats, RequestSpecDecodeMetrics
|
||||
from vllm.v1.structured_output.request import StructuredOutputRequest
|
||||
from vllm.v1.utils import ConstantList
|
||||
@@ -78,6 +79,7 @@ class Request:
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
abort_immediately: bool = False,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.client_index = client_index
|
||||
@@ -193,6 +195,7 @@ class Request:
|
||||
# trace_headers
|
||||
self.trace_headers = trace_headers
|
||||
self.session_id = session_id
|
||||
self.kv_hints = kv_hints
|
||||
|
||||
# True if this request is scheduled as a non-final prefill chunk.
|
||||
self.is_prefill_chunk = False
|
||||
@@ -260,6 +263,7 @@ class Request:
|
||||
block_hasher=block_hasher,
|
||||
resumable=request.resumable,
|
||||
session_id=request.session_id,
|
||||
kv_hints=request.kv_hints,
|
||||
reasoning_ended=request.reasoning_ended,
|
||||
reasoning_parser_kwargs=request.reasoning_parser_kwargs,
|
||||
abort_immediately=request.abort_immediately,
|
||||
|
||||
Reference in New Issue
Block a user