[Rust Frontend] Introduce parser-owned output grammar interfaces (#55269)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-09-22 05:35:07 +00:00
committed by GitHub
co-authored by Claude Fable 5.1
parent f84325c48c
commit d50723df04
34 changed files with 814 additions and 698 deletions
+21
View File
@@ -104,6 +104,27 @@ impl TextRequestProcessor {
Ok(prompt_token_ids)
}
/// Tokenize and truncate the prompt in place, so that later preparation
/// reuses the final token IDs instead of tokenizing again.
///
/// Frontends that need the final prompt tokens before submission (for
/// example, to initialize a prompt-aware output parser) call this first
/// and then hand the same request to [`TextLlm::generate`].
pub fn tokenize_in_place<'r>(&self, request: &'r mut TextRequest) -> Result<&'r [u32]> {
request.validate()?;
let prompt_token_ids = self.prepare_prompt_tokens(
take(&mut request.prompt),
request.add_special_tokens,
request.prompt_truncation.take(),
request.sampling_params.max_tokens,
)?;
request.prompt = Prompt::TokenIds(prompt_token_ids);
match &request.prompt {
Prompt::TokenIds(prompt_token_ids) => Ok(prompt_token_ids),
Prompt::Text(_) => unreachable!("prompt was just replaced with token IDs"),
}
}
/// Prepare one request's prompt tokens without generation-specific lowering.
pub fn tokenize(&self, request: TextRequest) -> Result<Vec<u32>> {
request.validate()?;
+2 -1
View File
@@ -61,7 +61,8 @@ pub fn lower_text_request(
data_parallel_rank: request.data_parallel_rank,
session_id: request.session_id.clone(),
kv_hints: request.kv_hints.clone(),
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
reasoning_parser_kwargs: Some(request.reasoning_parser_kwargs.clone()),
reasoning_ended: request.reasoning_ended,
lora_request: request.lora_request.clone(),
arrival_time: request.arrival_time,
trace_headers: None,
+14 -4
View File
@@ -215,10 +215,19 @@ pub struct TextRequest {
/// Optional orchestrator-originated KV hints.
#[serde(default)]
pub kv_hints: Option<KvHintsEnvelope>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
/// Reasoning-parser kwargs forwarded to engine-side structured output
/// logic. The engine consults them only when it owns grammar activation;
/// see [`Self::reasoning_ended`].
#[serde(default)]
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
pub reasoning_parser_kwargs: ReasoningParserKwargs,
/// Optional engine reasoning-gate override selected by a higher-level frontend.
///
/// `Some(true)` means the structured output grammar covers reasoning from
/// the first generated token, so the engine masks and advances immediately
/// instead of waiting for its reasoning parser. Unset for final-output-only
/// grammars and for requests without a grammar.
#[serde(default)]
pub reasoning_ended: Option<bool>,
/// LoRA adapter selected for this request.
#[serde(default)]
pub lora_request: Option<LoraRequest>,
@@ -247,7 +256,8 @@ impl TextRequest {
data_parallel_rank: None,
session_id: None,
kv_hints: None,
reasoning_parser_kwargs: None,
reasoning_parser_kwargs: Default::default(),
reasoning_ended: None,
lora_request: None,
arrival_time: None,
}