[Rust Frontend] Carry reasoning controls over gRPC (#59836)

Signed-off-by: Alec Flowers <aflowers@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alec
2026-10-09 11:11:19 +08:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent a1cd0ec650
commit 51ef239768
5 changed files with 108 additions and 5 deletions
+1 -1
View File
@@ -5947,7 +5947,7 @@ dependencies = [
[[package]]
name = "vllm-proto"
version = "0.4.0"
version = "0.5.0"
dependencies = [
"prost",
"prost-types",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "vllm-proto"
version = "0.4.0"
version = "0.5.0"
edition.workspace = true
license.workspace = true
description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend"
+1 -1
View File
@@ -16,7 +16,7 @@ the schemas on pull requests, including on forks.
```toml
[dependencies]
vllm-proto = "0.4"
vllm-proto = "0.5"
```
For example, import `vllm_proto::inference_client::InferenceClient` or
+22
View File
@@ -58,6 +58,25 @@ message GenerateRequest {
optional bool watermarking = 16;
KVHintsEnvelope kv_hints = 17;
// Transitional, to be deprecated: only consulted when the engine runs its own
// reasoning parser and the structured-output constraint covers only the
// output after reasoning. Prefer a constraint that covers the full output
// from the first generated token.
EngineReasoningGate engine_reasoning_gate = 18;
}
// Inputs to the engine's reasoning gate, which keeps a structured-output
// constraint inactive until the engine's reasoning parser detects the end of
// reasoning.
message EngineReasoningGate {
// Omitted: the engine infers the initial state from the prompt.
// true: apply the constraint from the first generated token.
// false: generation starts inside reasoning.
optional bool reasoning_ended = 1;
// Chat template kwargs used to render the prompt, so the engine's reasoning
// parser observes the same thinking mode.
google.protobuf.Struct chat_template_kwargs = 2;
}
message KVHintsEnvelope {
@@ -118,6 +137,9 @@ message StoppingCriteria {
bool include_stop_strings = 5;
bool ignore_eos = 6;
// Omitted or -1 disables the budget; zero immediately closes reasoning.
optional int64 thinking_token_budget = 7;
}
message ResponseOptions {
+83 -2
View File
@@ -9,6 +9,7 @@ use tonic::Status;
use uuid::Uuid;
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use vllm_text::{
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, PromptTruncation,
@@ -84,6 +85,7 @@ pub fn to_text_request(
};
let session_id = req.session_id.filter(|s| !s.is_empty());
let kv_hints = req.kv_hints.map(kv_hints_from_proto);
let reasoning_gate = req.engine_reasoning_gate.unwrap_or_default();
let sampling = req.sampling.as_ref();
let decoding = req.decoding.as_ref();
@@ -137,8 +139,15 @@ pub fn to_text_request(
data_parallel_rank: None,
session_id,
kv_hints,
reasoning_parser_kwargs: Default::default(),
reasoning_ended: None,
reasoning_parser_kwargs: ReasoningParserKwargs {
chat_template_kwargs: reasoning_gate
.chat_template_kwargs
.map(|kwargs| {
kwargs.fields.iter().map(|(k, v)| (k.clone(), proto_value_to_json(v))).collect()
})
.unwrap_or_default(),
},
reasoning_ended: reasoning_gate.reasoning_ended,
lora_request: None,
arrival_time: None,
})
@@ -213,6 +222,7 @@ fn build_sampling_params(
params.stop_token_ids = Some(s.stop_token_ids.clone());
}
params.ignore_eos = s.ignore_eos;
params.thinking_token_budget = s.thinking_token_budget;
}
// ResponseOptions → logprobs
@@ -611,6 +621,77 @@ mod tests {
}
}
#[test]
fn grpc_reasoning_controls_reach_engine_request_with_presence_intact() {
let chat_template_kwargs =
serde_json::json!({"enable_thinking": false, "reasoning_effort": "high"});
for (gate, budget) in [
(None, None),
(
Some(pb::EngineReasoningGate {
reasoning_ended: Some(false),
chat_template_kwargs: json_to_proto_struct(&chat_template_kwargs),
}),
Some(0),
),
(
Some(pb::EngineReasoningGate {
reasoning_ended: Some(true),
chat_template_kwargs: None,
}),
Some(128),
),
(Some(pb::EngineReasoningGate::default()), Some(-1)),
] {
let expected = (
serde_json::json!({
"chat_template_kwargs": gate
.as_ref()
.and_then(|gate| gate.chat_template_kwargs.as_ref())
.map_or_else(|| serde_json::json!({}), |_| chat_template_kwargs.clone()),
}),
gate.as_ref().and_then(|gate| gate.reasoning_ended),
budget.filter(|v| *v >= 0).map(|v| v as u64),
);
let request = pb::GenerateRequest {
engine_reasoning_gate: gate,
stopping: Some(pb::StoppingCriteria {
thinking_token_budget: budget,
..Default::default()
}),
..base_request()
};
let encoded = request.encode_to_vec();
for stream in [false, true] {
let request = pb::GenerateRequest::decode(encoded.as_slice()).unwrap();
let text = to_text_request(request, stream, &["test-model".to_string()])
.expect("reasoning controls accepted");
let engine = vllm_text::lower_text_request(
text,
vec![1],
SamplingHints::default(),
SamplingLimits {
max_model_len: 256,
max_logprobs: 20,
model_vocab_size: 512,
tokenizer_vocab_size: 512,
},
&TestTokenizer::new(),
)
.expect("reasoning controls lower to engine")
.generate_request;
assert_eq!(
(
serde_json::to_value(engine.reasoning_parser_kwargs).unwrap(),
engine.reasoning_ended,
engine.sampling_params.thinking_token_budget,
),
expected,
);
}
}
}
#[test]
fn watermarking_defaults_and_opt_out_survive_protobuf_conversion() {
for watermarking in [None, Some(true), Some(false)] {