feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)
Signed-off-by: Luis Robaina <luis@protopia.ai> Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com> Signed-off-by: LuisRobaina <luis@protopia.ai> Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
@@ -256,8 +256,10 @@ async def test_multi_abort(output_kind: RequestOutputKind):
|
||||
)
|
||||
)
|
||||
|
||||
# Let requests start
|
||||
await asyncio.sleep(0.5)
|
||||
# Let requests start generating, use a longer sleep to ensure all
|
||||
# requests have exited prefill and produced at least one
|
||||
# decode token before we abort.
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Use multi-abort to abort multiple requests at once
|
||||
abort_request_ids = [request_ids[i] for i in REQUEST_IDS_TO_ABORT]
|
||||
@@ -369,9 +371,10 @@ async def test_mid_stream_cancellation(
|
||||
# Wait for all tasks to complete
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
# Verify all tasks were cancelled at the expected point
|
||||
# Verify all tasks were cancelled at the expected point.
|
||||
# Uses >= because the cancel check is `count >= cancel_after`.
|
||||
for num_generated_tokens, request_id in results:
|
||||
assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
|
||||
assert num_generated_tokens >= NUM_EXPECTED_TOKENS, (
|
||||
f"{request_id} generated {num_generated_tokens} tokens but "
|
||||
f"expected to cancel after {NUM_EXPECTED_TOKENS}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user