Convert online APIs to use Renderer (#34084)

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
Co-authored-by: Reagan Lee <“reaganjlee@gmail.com”>
This commit is contained in:
Reagan Lee
2026-02-10 19:44:31 +00:00
committed by GitHub
co-authored by Reagan Lee
parent 33bcd3dc3b
commit fdd6f2ad58
2 changed files with 33 additions and 9 deletions
@@ -471,15 +471,31 @@ class OpenAISpeechToText(OpenAIServing):
lora_request=lora_request,
)
list_result_generator = [
self.engine_client.generate(
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
list_result_generator = []
for i, prompt in enumerate(prompts):
request_id_item = f"{request_id}_{i}"
engine_request = self.input_processor.process_inputs(
request_id_item,
prompt,
sampling_params,
f"{request_id}_{i}",
lora_request=lora_request,
trace_headers=trace_headers,
priority=0,
)
list_result_generator.append(
self.engine_client.generate(
engine_request,
sampling_params,
request_id_item,
lora_request=lora_request,
)
)
for i, prompt in enumerate(prompts)
]
except ValueError as e:
return self.create_error_response(e)