[Rust Frontend] Carry reasoning controls over gRPC (#59836)
Signed-off-by: Alec Flowers <aflowers@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
a1cd0ec650
commit
51ef239768
Generated
+1
-1
@@ -5947,7 +5947,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "vllm-proto"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
dependencies = [
|
||||
"prost",
|
||||
"prost-types",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "vllm-proto"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend"
|
||||
|
||||
@@ -16,7 +16,7 @@ the schemas on pull requests, including on forks.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
vllm-proto = "0.4"
|
||||
vllm-proto = "0.5"
|
||||
```
|
||||
|
||||
For example, import `vllm_proto::inference_client::InferenceClient` or
|
||||
|
||||
@@ -58,6 +58,25 @@ message GenerateRequest {
|
||||
optional bool watermarking = 16;
|
||||
|
||||
KVHintsEnvelope kv_hints = 17;
|
||||
|
||||
// Transitional, to be deprecated: only consulted when the engine runs its own
|
||||
// reasoning parser and the structured-output constraint covers only the
|
||||
// output after reasoning. Prefer a constraint that covers the full output
|
||||
// from the first generated token.
|
||||
EngineReasoningGate engine_reasoning_gate = 18;
|
||||
}
|
||||
|
||||
// Inputs to the engine's reasoning gate, which keeps a structured-output
|
||||
// constraint inactive until the engine's reasoning parser detects the end of
|
||||
// reasoning.
|
||||
message EngineReasoningGate {
|
||||
// Omitted: the engine infers the initial state from the prompt.
|
||||
// true: apply the constraint from the first generated token.
|
||||
// false: generation starts inside reasoning.
|
||||
optional bool reasoning_ended = 1;
|
||||
// Chat template kwargs used to render the prompt, so the engine's reasoning
|
||||
// parser observes the same thinking mode.
|
||||
google.protobuf.Struct chat_template_kwargs = 2;
|
||||
}
|
||||
|
||||
message KVHintsEnvelope {
|
||||
@@ -118,6 +137,9 @@ message StoppingCriteria {
|
||||
bool include_stop_strings = 5;
|
||||
|
||||
bool ignore_eos = 6;
|
||||
|
||||
// Omitted or -1 disables the budget; zero immediately closes reasoning.
|
||||
optional int64 thinking_token_budget = 7;
|
||||
}
|
||||
|
||||
message ResponseOptions {
|
||||
|
||||
@@ -9,6 +9,7 @@ use tonic::Status;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, PromptTruncation,
|
||||
@@ -84,6 +85,7 @@ pub fn to_text_request(
|
||||
};
|
||||
let session_id = req.session_id.filter(|s| !s.is_empty());
|
||||
let kv_hints = req.kv_hints.map(kv_hints_from_proto);
|
||||
let reasoning_gate = req.engine_reasoning_gate.unwrap_or_default();
|
||||
|
||||
let sampling = req.sampling.as_ref();
|
||||
let decoding = req.decoding.as_ref();
|
||||
@@ -137,8 +139,15 @@ pub fn to_text_request(
|
||||
data_parallel_rank: None,
|
||||
session_id,
|
||||
kv_hints,
|
||||
reasoning_parser_kwargs: Default::default(),
|
||||
reasoning_ended: None,
|
||||
reasoning_parser_kwargs: ReasoningParserKwargs {
|
||||
chat_template_kwargs: reasoning_gate
|
||||
.chat_template_kwargs
|
||||
.map(|kwargs| {
|
||||
kwargs.fields.iter().map(|(k, v)| (k.clone(), proto_value_to_json(v))).collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
},
|
||||
reasoning_ended: reasoning_gate.reasoning_ended,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
})
|
||||
@@ -213,6 +222,7 @@ fn build_sampling_params(
|
||||
params.stop_token_ids = Some(s.stop_token_ids.clone());
|
||||
}
|
||||
params.ignore_eos = s.ignore_eos;
|
||||
params.thinking_token_budget = s.thinking_token_budget;
|
||||
}
|
||||
|
||||
// ResponseOptions → logprobs
|
||||
@@ -611,6 +621,77 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grpc_reasoning_controls_reach_engine_request_with_presence_intact() {
|
||||
let chat_template_kwargs =
|
||||
serde_json::json!({"enable_thinking": false, "reasoning_effort": "high"});
|
||||
for (gate, budget) in [
|
||||
(None, None),
|
||||
(
|
||||
Some(pb::EngineReasoningGate {
|
||||
reasoning_ended: Some(false),
|
||||
chat_template_kwargs: json_to_proto_struct(&chat_template_kwargs),
|
||||
}),
|
||||
Some(0),
|
||||
),
|
||||
(
|
||||
Some(pb::EngineReasoningGate {
|
||||
reasoning_ended: Some(true),
|
||||
chat_template_kwargs: None,
|
||||
}),
|
||||
Some(128),
|
||||
),
|
||||
(Some(pb::EngineReasoningGate::default()), Some(-1)),
|
||||
] {
|
||||
let expected = (
|
||||
serde_json::json!({
|
||||
"chat_template_kwargs": gate
|
||||
.as_ref()
|
||||
.and_then(|gate| gate.chat_template_kwargs.as_ref())
|
||||
.map_or_else(|| serde_json::json!({}), |_| chat_template_kwargs.clone()),
|
||||
}),
|
||||
gate.as_ref().and_then(|gate| gate.reasoning_ended),
|
||||
budget.filter(|v| *v >= 0).map(|v| v as u64),
|
||||
);
|
||||
let request = pb::GenerateRequest {
|
||||
engine_reasoning_gate: gate,
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
thinking_token_budget: budget,
|
||||
..Default::default()
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
let encoded = request.encode_to_vec();
|
||||
for stream in [false, true] {
|
||||
let request = pb::GenerateRequest::decode(encoded.as_slice()).unwrap();
|
||||
let text = to_text_request(request, stream, &["test-model".to_string()])
|
||||
.expect("reasoning controls accepted");
|
||||
let engine = vllm_text::lower_text_request(
|
||||
text,
|
||||
vec![1],
|
||||
SamplingHints::default(),
|
||||
SamplingLimits {
|
||||
max_model_len: 256,
|
||||
max_logprobs: 20,
|
||||
model_vocab_size: 512,
|
||||
tokenizer_vocab_size: 512,
|
||||
},
|
||||
&TestTokenizer::new(),
|
||||
)
|
||||
.expect("reasoning controls lower to engine")
|
||||
.generate_request;
|
||||
assert_eq!(
|
||||
(
|
||||
serde_json::to_value(engine.reasoning_parser_kwargs).unwrap(),
|
||||
engine.reasoning_ended,
|
||||
engine.sampling_params.thinking_token_budget,
|
||||
),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watermarking_defaults_and_opt_out_survive_protobuf_conversion() {
|
||||
for watermarking in [None, Some(true), Some(false)] {
|
||||
|
||||
Reference in New Issue
Block a user