[Perf] Add skip_clone to SamplingParams for internal request handling (#31041)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2025-12-24 14:35:57 -08:00
committed by GitHub
parent 09dc7c690c
commit bc5ef333e0
5 changed files with 25 additions and 2 deletions
+2 -1
View File
@@ -60,7 +60,8 @@ async def generate(request: Request) -> Response:
async def _generate(request_dict: dict, raw_request: Request) -> Response:
prompt = request_dict.pop("prompt")
stream = request_dict.pop("stream", False)
sampling_params = SamplingParams(**request_dict)
# Since SamplingParams is created fresh per request, safe to skip clone
sampling_params = SamplingParams(**request_dict, skip_clone=True)
request_id = random_uuid()
assert engine is not None