diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 331ebfa8df..c7b3d66e4c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "vllm-proto" -version = "0.4.0" +version = "0.5.0" dependencies = [ "prost", "prost-types", diff --git a/rust/proto/Cargo.toml b/rust/proto/Cargo.toml index 492cb5a06d..c2e32d43a7 100644 --- a/rust/proto/Cargo.toml +++ b/rust/proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vllm-proto" -version = "0.4.0" +version = "0.5.0" edition.workspace = true license.workspace = true description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend" diff --git a/rust/proto/README.md b/rust/proto/README.md index d312e27aca..3a60c05dfd 100644 --- a/rust/proto/README.md +++ b/rust/proto/README.md @@ -16,7 +16,7 @@ the schemas on pull requests, including on forks. ```toml [dependencies] -vllm-proto = "0.4" +vllm-proto = "0.5" ``` For example, import `vllm_proto::inference_client::InferenceClient` or diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 79c1d0a147..2daacbfc35 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -58,6 +58,25 @@ message GenerateRequest { optional bool watermarking = 16; KVHintsEnvelope kv_hints = 17; + + // Transitional, to be deprecated: only consulted when the engine runs its own + // reasoning parser and the structured-output constraint covers only the + // output after reasoning. Prefer a constraint that covers the full output + // from the first generated token. + EngineReasoningGate engine_reasoning_gate = 18; +} + +// Inputs to the engine's reasoning gate, which keeps a structured-output +// constraint inactive until the engine's reasoning parser detects the end of +// reasoning. +message EngineReasoningGate { + // Omitted: the engine infers the initial state from the prompt. + // true: apply the constraint from the first generated token. + // false: generation starts inside reasoning. + optional bool reasoning_ended = 1; + // Chat template kwargs used to render the prompt, so the engine's reasoning + // parser observes the same thinking mode. + google.protobuf.Struct chat_template_kwargs = 2; } message KVHintsEnvelope { @@ -118,6 +137,9 @@ message StoppingCriteria { bool include_stop_strings = 5; bool ignore_eos = 6; + + // Omitted or -1 disables the budget; zero immediately closes reasoning. + optional int64 thinking_token_budget = 7; } message ResponseOptions { diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index a409cc7f79..d495fea485 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -9,6 +9,7 @@ use tonic::Status; use uuid::Uuid; use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope}; use vllm_engine_core_client::protocol::output::StopReason; +use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; use vllm_text::{ DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, PromptTruncation, @@ -84,6 +85,7 @@ pub fn to_text_request( }; let session_id = req.session_id.filter(|s| !s.is_empty()); let kv_hints = req.kv_hints.map(kv_hints_from_proto); + let reasoning_gate = req.engine_reasoning_gate.unwrap_or_default(); let sampling = req.sampling.as_ref(); let decoding = req.decoding.as_ref(); @@ -137,8 +139,15 @@ pub fn to_text_request( data_parallel_rank: None, session_id, kv_hints, - reasoning_parser_kwargs: Default::default(), - reasoning_ended: None, + reasoning_parser_kwargs: ReasoningParserKwargs { + chat_template_kwargs: reasoning_gate + .chat_template_kwargs + .map(|kwargs| { + kwargs.fields.iter().map(|(k, v)| (k.clone(), proto_value_to_json(v))).collect() + }) + .unwrap_or_default(), + }, + reasoning_ended: reasoning_gate.reasoning_ended, lora_request: None, arrival_time: None, }) @@ -213,6 +222,7 @@ fn build_sampling_params( params.stop_token_ids = Some(s.stop_token_ids.clone()); } params.ignore_eos = s.ignore_eos; + params.thinking_token_budget = s.thinking_token_budget; } // ResponseOptions → logprobs @@ -611,6 +621,77 @@ mod tests { } } + #[test] + fn grpc_reasoning_controls_reach_engine_request_with_presence_intact() { + let chat_template_kwargs = + serde_json::json!({"enable_thinking": false, "reasoning_effort": "high"}); + for (gate, budget) in [ + (None, None), + ( + Some(pb::EngineReasoningGate { + reasoning_ended: Some(false), + chat_template_kwargs: json_to_proto_struct(&chat_template_kwargs), + }), + Some(0), + ), + ( + Some(pb::EngineReasoningGate { + reasoning_ended: Some(true), + chat_template_kwargs: None, + }), + Some(128), + ), + (Some(pb::EngineReasoningGate::default()), Some(-1)), + ] { + let expected = ( + serde_json::json!({ + "chat_template_kwargs": gate + .as_ref() + .and_then(|gate| gate.chat_template_kwargs.as_ref()) + .map_or_else(|| serde_json::json!({}), |_| chat_template_kwargs.clone()), + }), + gate.as_ref().and_then(|gate| gate.reasoning_ended), + budget.filter(|v| *v >= 0).map(|v| v as u64), + ); + let request = pb::GenerateRequest { + engine_reasoning_gate: gate, + stopping: Some(pb::StoppingCriteria { + thinking_token_budget: budget, + ..Default::default() + }), + ..base_request() + }; + let encoded = request.encode_to_vec(); + for stream in [false, true] { + let request = pb::GenerateRequest::decode(encoded.as_slice()).unwrap(); + let text = to_text_request(request, stream, &["test-model".to_string()]) + .expect("reasoning controls accepted"); + let engine = vllm_text::lower_text_request( + text, + vec![1], + SamplingHints::default(), + SamplingLimits { + max_model_len: 256, + max_logprobs: 20, + model_vocab_size: 512, + tokenizer_vocab_size: 512, + }, + &TestTokenizer::new(), + ) + .expect("reasoning controls lower to engine") + .generate_request; + assert_eq!( + ( + serde_json::to_value(engine.reasoning_parser_kwargs).unwrap(), + engine.reasoning_ended, + engine.sampling_params.thinking_token_budget, + ), + expected, + ); + } + } + } + #[test] fn watermarking_defaults_and_opt_out_survive_protobuf_conversion() { for watermarking in [None, Some(true), Some(false)] {