[Feat][CLI] enforce-include-usage (#19695)

Signed-off-by: Max Wittig <max.wittig@siemens.com>
This commit is contained in:
Max Wittig
2025-06-25 01:43:04 -04:00
committed by GitHub
parent 879f69bed3
commit f59fc60fb3
5 changed files with 34 additions and 9 deletions
@@ -52,12 +52,14 @@ class OpenAIServingCompletion(OpenAIServing):
*,
request_logger: Optional[RequestLogger],
return_tokens_as_token_ids: bool = False,
enable_force_include_usage: bool = False,
):
super().__init__(engine_client=engine_client,
model_config=model_config,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids)
return_tokens_as_token_ids=return_tokens_as_token_ids,
enable_force_include_usage=enable_force_include_usage)
self.default_sampling_params = (
self.model_config.get_diff_sampling_param())
if self.default_sampling_params:
@@ -227,7 +229,8 @@ class OpenAIServingCompletion(OpenAIServing):
model_name,
num_prompts=num_prompts,
tokenizer=tokenizer,
request_metadata=request_metadata)
request_metadata=request_metadata,
enable_force_include_usage=self.enable_force_include_usage)
# Non-streaming response
final_res_batch: list[Optional[RequestOutput]] = [None] * num_prompts
@@ -289,6 +292,7 @@ class OpenAIServingCompletion(OpenAIServing):
num_prompts: int,
tokenizer: AnyTokenizer,
request_metadata: RequestResponseMetadata,
enable_force_include_usage: bool,
) -> AsyncGenerator[str, None]:
num_choices = 1 if request.n is None else request.n
previous_text_lens = [0] * num_choices * num_prompts
@@ -298,7 +302,8 @@ class OpenAIServingCompletion(OpenAIServing):
stream_options = request.stream_options
if stream_options:
include_usage = stream_options.include_usage
include_usage = stream_options.include_usage or \
enable_force_include_usage
include_continuous_usage = include_usage and \
stream_options.continuous_usage_stats
else: