[Rust Frontend] Introduce parser-owned output grammar interfaces (#55269)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f84325c48c
commit
d50723df04
@@ -104,6 +104,27 @@ impl TextRequestProcessor {
|
||||
Ok(prompt_token_ids)
|
||||
}
|
||||
|
||||
/// Tokenize and truncate the prompt in place, so that later preparation
|
||||
/// reuses the final token IDs instead of tokenizing again.
|
||||
///
|
||||
/// Frontends that need the final prompt tokens before submission (for
|
||||
/// example, to initialize a prompt-aware output parser) call this first
|
||||
/// and then hand the same request to [`TextLlm::generate`].
|
||||
pub fn tokenize_in_place<'r>(&self, request: &'r mut TextRequest) -> Result<&'r [u32]> {
|
||||
request.validate()?;
|
||||
let prompt_token_ids = self.prepare_prompt_tokens(
|
||||
take(&mut request.prompt),
|
||||
request.add_special_tokens,
|
||||
request.prompt_truncation.take(),
|
||||
request.sampling_params.max_tokens,
|
||||
)?;
|
||||
request.prompt = Prompt::TokenIds(prompt_token_ids);
|
||||
match &request.prompt {
|
||||
Prompt::TokenIds(prompt_token_ids) => Ok(prompt_token_ids),
|
||||
Prompt::Text(_) => unreachable!("prompt was just replaced with token IDs"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepare one request's prompt tokens without generation-specific lowering.
|
||||
pub fn tokenize(&self, request: TextRequest) -> Result<Vec<u32>> {
|
||||
request.validate()?;
|
||||
|
||||
@@ -61,7 +61,8 @@ pub fn lower_text_request(
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
session_id: request.session_id.clone(),
|
||||
kv_hints: request.kv_hints.clone(),
|
||||
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
|
||||
reasoning_parser_kwargs: Some(request.reasoning_parser_kwargs.clone()),
|
||||
reasoning_ended: request.reasoning_ended,
|
||||
lora_request: request.lora_request.clone(),
|
||||
arrival_time: request.arrival_time,
|
||||
trace_headers: None,
|
||||
|
||||
@@ -215,10 +215,19 @@ pub struct TextRequest {
|
||||
/// Optional orchestrator-originated KV hints.
|
||||
#[serde(default)]
|
||||
pub kv_hints: Option<KvHintsEnvelope>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
/// Reasoning-parser kwargs forwarded to engine-side structured output
|
||||
/// logic. The engine consults them only when it owns grammar activation;
|
||||
/// see [`Self::reasoning_ended`].
|
||||
#[serde(default)]
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
pub reasoning_parser_kwargs: ReasoningParserKwargs,
|
||||
/// Optional engine reasoning-gate override selected by a higher-level frontend.
|
||||
///
|
||||
/// `Some(true)` means the structured output grammar covers reasoning from
|
||||
/// the first generated token, so the engine masks and advances immediately
|
||||
/// instead of waiting for its reasoning parser. Unset for final-output-only
|
||||
/// grammars and for requests without a grammar.
|
||||
#[serde(default)]
|
||||
pub reasoning_ended: Option<bool>,
|
||||
/// LoRA adapter selected for this request.
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<LoraRequest>,
|
||||
@@ -247,7 +256,8 @@ impl TextRequest {
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
reasoning_parser_kwargs: Default::default(),
|
||||
reasoning_ended: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user