[Rust Frontend] Extract request preparation from the inference path (#49045)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
+108
-72
@@ -38,32 +38,22 @@ trait_set! {
|
||||
pub trait TextOutputStream = Stream<Item = Result<DecodedTextEvent>> + Send + 'static;
|
||||
}
|
||||
|
||||
/// Raw text facade above [`Llm`].
|
||||
///
|
||||
/// This layer stays below chat semantics: prompt text or prompt token IDs flow
|
||||
/// in, decoded text deltas and terminal metadata flow out.
|
||||
pub struct TextLlm {
|
||||
/// Generate-only client owned by this text facade.
|
||||
llm: Llm,
|
||||
/// Text request preparation shared by inference and render-only frontends.
|
||||
pub struct TextRequestProcessor {
|
||||
/// Tokenizer/model metadata backend responsible for prompt encode/decode
|
||||
/// and sampling hints.
|
||||
backend: DynTextBackend,
|
||||
/// Runtime context window size reported by the engine startup handshake.
|
||||
/// Render-only frontends supply the downstream engine's effective value.
|
||||
max_model_len: u32,
|
||||
/// Maximum number of top log probabilities accepted by this text facade.
|
||||
max_logprobs: i32,
|
||||
}
|
||||
|
||||
impl TextLlm {
|
||||
/// Create a new text-generation facade from a shared LLM client plus a text
|
||||
/// backend.
|
||||
pub fn new(llm: Llm, backend: DynTextBackend) -> Self {
|
||||
// The engine-reported value reflects the post-profiling, auto-fitted
|
||||
// KV cache limit used at runtime.
|
||||
let max_model_len = llm.engine_core_client().max_model_len();
|
||||
|
||||
impl TextRequestProcessor {
|
||||
/// Create a processor with the effective model context length.
|
||||
pub fn new(backend: DynTextBackend, max_model_len: u32) -> Self {
|
||||
Self {
|
||||
llm,
|
||||
backend,
|
||||
max_model_len,
|
||||
max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS,
|
||||
@@ -78,61 +68,18 @@ impl TextLlm {
|
||||
self
|
||||
}
|
||||
|
||||
/// Return the backend model ID.
|
||||
pub fn model_id(&self) -> &str {
|
||||
self.backend.model_id()
|
||||
}
|
||||
|
||||
/// Expose the underlying engine-core client for low-level utility/admin
|
||||
/// calls.
|
||||
pub fn engine_core_client(&self) -> &EngineCoreClient {
|
||||
self.llm.engine_core_client()
|
||||
}
|
||||
|
||||
/// Return the tokenizer used by this text backend.
|
||||
/// Return the tokenizer used by this processor.
|
||||
pub fn tokenizer(&self) -> DynTokenizer {
|
||||
self.backend.tokenizer()
|
||||
}
|
||||
|
||||
/// Tokenizer vocabulary size (the number of tokens the tokenizer knows),
|
||||
/// used to bound `allowed_token_ids` like the Python frontend `len(tokenizer)`.
|
||||
pub fn tokenizer_vocab_size(&self) -> usize {
|
||||
self.backend.tokenizer_vocab_size()
|
||||
/// Return the effective model context length.
|
||||
pub fn max_model_len(&self) -> u32 {
|
||||
self.max_model_len
|
||||
}
|
||||
|
||||
/// Model vocabulary size from the model config, used to bound generated
|
||||
/// token IDs and logits-domain sampling controls.
|
||||
pub fn model_vocab_size(&self) -> usize {
|
||||
self.backend.model_vocab_size()
|
||||
}
|
||||
|
||||
/// Tokenize if needed, lower to a generate request, and return the raw
|
||||
/// token stream.
|
||||
pub async fn generate_raw(&self, request: TextRequest) -> Result<GenerateOutputStream> {
|
||||
let (_, raw_stream) = self.generate_inner(request).await?;
|
||||
Ok(raw_stream)
|
||||
}
|
||||
|
||||
/// Tokenize if needed, lower to a generate request, and stream
|
||||
/// incrementally decoded text.
|
||||
pub async fn generate(&self, request: TextRequest) -> Result<impl TextOutputStream> {
|
||||
let (text_request, raw_stream) = self.generate_inner(request).await?;
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let decoded_stream = output::decoded_text_event_stream(
|
||||
text_request.request_id,
|
||||
tokenizer,
|
||||
raw_stream,
|
||||
text_request.decode_options,
|
||||
text_request.intermediate,
|
||||
);
|
||||
|
||||
Ok(decoded_stream)
|
||||
}
|
||||
|
||||
async fn generate_inner(
|
||||
&self,
|
||||
mut request: TextRequest,
|
||||
) -> Result<(TextRequest, GenerateOutputStream)> {
|
||||
/// Tokenize and lower one request without submitting it to an engine.
|
||||
pub fn prepare(&self, mut request: TextRequest) -> Result<PreparedTextRequest> {
|
||||
request.validate()?;
|
||||
|
||||
if request.arrival_time.is_none() {
|
||||
@@ -146,7 +93,6 @@ impl TextLlm {
|
||||
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
|
||||
Prompt::TokenIds(token_ids) => token_ids,
|
||||
};
|
||||
|
||||
let sampling_hints = self.backend.sampling_hints()?;
|
||||
let sampling_limits = SamplingLimits {
|
||||
max_model_len: self.max_model_len,
|
||||
@@ -155,16 +101,106 @@ impl TextLlm {
|
||||
tokenizer_vocab_size: self.backend.tokenizer_vocab_size(),
|
||||
};
|
||||
|
||||
let PreparedTextRequest {
|
||||
text_request,
|
||||
generate_request,
|
||||
} = lower_text_request(
|
||||
lower_text_request(
|
||||
request,
|
||||
prompt_token_ids,
|
||||
sampling_hints,
|
||||
sampling_limits,
|
||||
&*tokenizer,
|
||||
)?;
|
||||
tokenizer.as_ref(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw text facade above [`Llm`].
|
||||
///
|
||||
/// This layer stays below chat semantics: prompt text or prompt token IDs flow
|
||||
/// in, decoded text deltas and terminal metadata flow out.
|
||||
pub struct TextLlm {
|
||||
/// Generate-only client owned by this text facade.
|
||||
llm: Llm,
|
||||
/// Shared engine-free request preparation.
|
||||
processor: TextRequestProcessor,
|
||||
}
|
||||
|
||||
impl TextLlm {
|
||||
/// Create a new text-generation facade from a shared LLM client plus a text
|
||||
/// backend.
|
||||
pub fn new(llm: Llm, backend: DynTextBackend) -> Self {
|
||||
// The engine-reported value reflects the post-profiling, auto-fitted
|
||||
// KV cache limit used at runtime.
|
||||
let max_model_len = llm.engine_core_client().max_model_len();
|
||||
|
||||
Self {
|
||||
llm,
|
||||
processor: TextRequestProcessor::new(backend, max_model_len),
|
||||
}
|
||||
}
|
||||
|
||||
/// Override the maximum accepted logprobs count.
|
||||
pub fn with_max_logprobs(mut self, max_logprobs: Option<i32>) -> Self {
|
||||
self.processor = self.processor.with_max_logprobs(max_logprobs);
|
||||
self
|
||||
}
|
||||
|
||||
/// Return the backend model ID.
|
||||
pub fn model_id(&self) -> &str {
|
||||
self.processor.backend.model_id()
|
||||
}
|
||||
|
||||
/// Expose the underlying engine-core client for low-level utility/admin
|
||||
/// calls.
|
||||
pub fn engine_core_client(&self) -> &EngineCoreClient {
|
||||
self.llm.engine_core_client()
|
||||
}
|
||||
|
||||
/// Return the tokenizer used by this text backend.
|
||||
pub fn tokenizer(&self) -> DynTokenizer {
|
||||
self.processor.tokenizer()
|
||||
}
|
||||
|
||||
/// Tokenizer vocabulary size (the number of tokens the tokenizer knows),
|
||||
/// used to bound `allowed_token_ids` like the Python frontend `len(tokenizer)`.
|
||||
pub fn tokenizer_vocab_size(&self) -> usize {
|
||||
self.processor.backend.tokenizer_vocab_size()
|
||||
}
|
||||
|
||||
/// Model vocabulary size from the model config, used to bound generated
|
||||
/// token IDs and logits-domain sampling controls.
|
||||
pub fn model_vocab_size(&self) -> usize {
|
||||
self.processor.backend.model_vocab_size()
|
||||
}
|
||||
|
||||
/// Tokenize if needed, lower to a generate request, and return the raw
|
||||
/// token stream.
|
||||
pub async fn generate_raw(&self, request: TextRequest) -> Result<GenerateOutputStream> {
|
||||
let (_, raw_stream) = self.generate_inner(request).await?;
|
||||
Ok(raw_stream)
|
||||
}
|
||||
|
||||
/// Tokenize if needed, lower to a generate request, and stream
|
||||
/// incrementally decoded text.
|
||||
pub async fn generate(&self, request: TextRequest) -> Result<impl TextOutputStream> {
|
||||
let (text_request, raw_stream) = self.generate_inner(request).await?;
|
||||
let tokenizer = self.processor.tokenizer();
|
||||
let decoded_stream = output::decoded_text_event_stream(
|
||||
text_request.request_id,
|
||||
tokenizer,
|
||||
raw_stream,
|
||||
text_request.decode_options,
|
||||
text_request.intermediate,
|
||||
);
|
||||
|
||||
Ok(decoded_stream)
|
||||
}
|
||||
|
||||
async fn generate_inner(
|
||||
&self,
|
||||
request: TextRequest,
|
||||
) -> Result<(TextRequest, GenerateOutputStream)> {
|
||||
let PreparedTextRequest {
|
||||
text_request,
|
||||
generate_request,
|
||||
} = self.processor.prepare(request)?;
|
||||
|
||||
let raw_stream = self.llm.generate(generate_request).await?;
|
||||
Ok((text_request, raw_stream))
|
||||
|
||||
Reference in New Issue
Block a user