[BugFix] Fix duplicate id tool-call race condition (#29355)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-11-25 01:54:26 +00:00
committed by GitHub
parent b8328b49fb
commit a178a0b40b
2 changed files with 16 additions and 7 deletions
+7 -3
View File
@@ -1242,16 +1242,19 @@ class OpenAIServing:
):
prompt_text, _, _ = self._get_prompt_components(request_prompt)
orig_priority = priority
sub_request = 0
while True:
# Ensure that each sub-request has a unique request id.
sub_request_id = f"{request_id}_{sub_request}"
self._log_inputs(
request_id,
sub_request_id,
request_prompt,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = kwargs.get("trace_headers")
engine_request, tokenization_kwargs = await self._process_inputs(
request_id,
sub_request_id,
engine_prompt,
sampling_params,
lora_request=lora_request,
@@ -1262,7 +1265,7 @@ class OpenAIServing:
generator = self.engine_client.generate(
engine_request,
sampling_params,
request_id,
sub_request_id,
lora_request=lora_request,
priority=priority,
prompt_text=prompt_text,
@@ -1295,6 +1298,7 @@ class OpenAIServing:
sampling_params.max_tokens = self.max_model_len - len(prompt_token_ids)
# OPTIMIZATION
priority = orig_priority - 1
sub_request += 1
def _get_prompt_components(
self,