[Frontend] Consolidate scale out entrypoints (#44512)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-29 03:18:53 -07:00
committed by GitHub
parent eddfd4cf21
commit 3483240b7e
45 changed files with 422 additions and 289 deletions
+4
View File
@@ -845,10 +845,12 @@ steps:
source_file_dependencies: source_file_dependencies:
- vllm/ - vllm/
- tests/entrypoints/serve - tests/entrypoints/serve
- tests/entrypoints/scale_out
commands: commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
- pytest -v -s entrypoints/scale_out
- label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
@@ -2559,10 +2561,12 @@ steps:
source_file_dependencies: source_file_dependencies:
- vllm/ - vllm/
- tests/entrypoints/serve - tests/entrypoints/serve
- tests/entrypoints/scale_out
commands: commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
- pytest -v -s entrypoints/scale_out
- label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
+2
View File
@@ -40,10 +40,12 @@ steps:
source_file_dependencies: source_file_dependencies:
- vllm/ - vllm/
- tests/entrypoints/serve - tests/entrypoints/serve
- tests/entrypoints/scale_out
commands: commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
- pytest -v -s entrypoints/scale_out
mirror: mirror:
amd: amd:
device: mi325_1 device: mi325_1
+2 -2
View File
@@ -46,7 +46,7 @@ steps:
- vllm/v1/engine/ - vllm/v1/engine/
- tests/utils.py - tests/utils.py
# - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
- tests/entrypoints/serve/disagg/test_serving_tokens.py - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
- tests/entrypoints/serve/instrumentator/test_basic.py - tests/entrypoints/serve/instrumentator/test_basic.py
- tests/entrypoints/serve/instrumentator/test_metrics.py - tests/entrypoints/serve/instrumentator/test_metrics.py
# - tests/entrypoints/serve/dev/test_sleep.py # - tests/entrypoints/serve/dev/test_sleep.py
@@ -55,7 +55,7 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
- pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load" - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
- pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow" - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
- pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist" - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
# - pytest -v -s entrypoints/serve/dev/test_sleep.py # - pytest -v -s entrypoints/serve/dev/test_sleep.py
+2 -1
View File
@@ -13,5 +13,6 @@ vLLM's examples are organized into the following categories:
- **[`rl/`](../../examples/rl)** – Reinforcement learning examples. - **[`rl/`](../../examples/rl)** – Reinforcement learning examples.
- **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production. - **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production.
- **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray. - **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray.
- **[`disaggregated/`](../../examples/disaggregated)** – Examples for disaggregated serving (separate prefill and decode), including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery. - **[`disaggregated/`](../../examples/disaggregated)** – Examples for Disaggregated P/D (Prefill/Decoding) inference, including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
- **[`scale_out/`](../../examples/scale_out)** – Examples for Token In <> Token Out API Server.
- **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses). - **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses).
+2 -2
View File
@@ -119,9 +119,9 @@ For further details on profiling vLLM, please refer to [this page](../../contrib
- `/ping` - SageMaker health check - `/ping` - SageMaker health check
- `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints) - `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
## Disaggregated Everything ## Scale-Out APIs
### Tokens IN <> Tokens OUT ### Tokens IN <> Tokens OUT APIs
- `/inference/v1/generate` - Generate completions - `/inference/v1/generate` - Generate completions
- `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set) - `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)
@@ -17,10 +17,9 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import ServingRender from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.outputs import CompletionOutput, RequestOutput from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.async_llm import AsyncLLM
@@ -206,15 +205,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
chat_template=None, chat_template=None,
chat_template_content_format="auto", chat_template_content_format="auto",
) )
online_derenderer = OnlineDerenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_render = ServingRender(models, online_renderer, online_derenderer) serving_render = ServingRender(models, online_renderer)
async def _fake_preprocess_chat(*args, **kwargs): async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs # return conversation, engine_inputs
@@ -14,10 +14,9 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import ServingRender from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.outputs import CompletionOutput, RequestOutput from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.async_llm import AsyncLLM
@@ -191,15 +190,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
chat_template=None, chat_template=None,
chat_template_content_format="auto", chat_template_content_format="auto",
) )
online_derenderer = OnlineDerenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_render = ServingRender(models, online_renderer, online_derenderer) serving_render = ServingRender(models, online_renderer)
async def _fake_preprocess_chat(*args, **kwargs): async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs # return conversation, engine_inputs
@@ -3,7 +3,7 @@
"""Unit tests for the token-offsets request/response protocol wiring: """Unit tests for the token-offsets request/response protocol wiring:
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest`` the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
serialization boundary. End-to-end behavior is covered by serialization boundary. End-to-end behavior is covered by
``tests/entrypoints/serve/render/test_render.py``; plain Pydantic field ``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field
storage is not retested here. storage is not retested here.
""" """
@@ -12,7 +12,7 @@ from unittest.mock import Mock
from vllm.config import ModelConfig from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.sampling_params import SamplingParams from vllm.sampling_params import SamplingParams
@@ -12,11 +12,11 @@ from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.engine.protocol import StreamOptions from vllm.entrypoints.openai.engine.protocol import StreamOptions
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.disagg.protocol import ( from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest, GenerateRequest,
GenerateResponse, GenerateResponse,
) )
from vllm.entrypoints.serve.disagg.serving import ServingTokens from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers import renderer_from_config from vllm.renderers import renderer_from_config
@@ -1,14 +1,17 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Roundtrip tests for multimodal serde used by the disagg generate endpoint.""" """
Roundtrip tests for multimodal serde used by the
token_in_token_out generate endpoint.
"""
import torch import torch
from vllm.entrypoints.serve.disagg.mm_serde import ( from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item, decode_mm_kwargs_item,
encode_mm_kwargs_item, encode_mm_kwargs_item,
) )
from vllm.entrypoints.serve.disagg.protocol import ( from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
MultiModalFeatures, MultiModalFeatures,
PlaceholderRangeInfo, PlaceholderRangeInfo,
) )
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the disagg request/response protocol. """Unit tests for the token_in_token_out request/response protocol.
These tests intentionally avoid spinning up a server — they exercise the These tests intentionally avoid spinning up a server — they exercise the
pydantic validators on ``GenerateRequest`` directly so they run fast and pydantic validators on ``GenerateRequest`` directly so they run fast and
@@ -9,7 +9,7 @@ fail loudly if the validator semantics ever drift.
import json import json
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.sampling_params import SamplingParams from vllm.sampling_params import SamplingParams
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.entrypoints.serve.disagg.serving import ServingTokens from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob from vllm.logprobs import Logprob
+1 -1
View File
@@ -5,7 +5,7 @@
These exercise ``_tokenize_prompt`` (offset extraction + capability/MM These exercise ``_tokenize_prompt`` (offset extraction + capability/MM
gating) and the ``_tokenize_prompt -> _process_tokens -> TokensInput`` gating) and the ``_tokenize_prompt -> _process_tokens -> TokensInput``
forwarding chain. Endpoint-level coverage lives in forwarding chain. Endpoint-level coverage lives in
``tests/entrypoints/serve/render/test_render.py``. ``tests/entrypoints/scale_out/render/test_render.py``.
""" """
import pytest import pytest
-15
View File
@@ -66,7 +66,6 @@ async def init_generate_state(
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
from vllm.entrypoints.serve.disagg.serving import ServingTokens
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
# Applied before any serving class is constructed so that each one picks # Applied before any serving class is constructed so that each one picks
@@ -175,20 +174,6 @@ async def init_generate_state(
if "generate" in supported_tasks if "generate" in supported_tasks
else None else None
) )
state.serving_tokens = (
ServingTokens(
engine_client,
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_log_outputs=args.enable_log_outputs,
force_no_detokenize=args.tokens_only,
)
if "generate" in supported_tasks
else None
)
from .generative_scoring.serving import ServingGenerativeScoring from .generative_scoring.serving import ServingGenerativeScoring
+10 -23
View File
@@ -32,7 +32,6 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import ( from vllm.entrypoints.serve.utils.api_utils import (
@@ -208,12 +207,6 @@ def build_app(
register_generate_api_routers(app) register_generate_api_routers(app)
from vllm.entrypoints.serve.disagg.api_router import (
attach_router as attach_disagg_router,
)
attach_disagg_router(app)
from vllm.entrypoints.serve.elastic_ep.api_router import ( from vllm.entrypoints.serve.elastic_ep.api_router import (
attach_router as elastic_ep_attach_router, attach_router as elastic_ep_attach_router,
) )
@@ -221,11 +214,9 @@ def build_app(
elastic_ep_attach_router(app) elastic_ep_attach_router(app)
if "generate" in supported_tasks or "render" in supported_tasks: if "generate" in supported_tasks or "render" in supported_tasks:
from vllm.entrypoints.serve.render.api_router import ( from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
attach_router as attach_render_router,
)
attach_render_router(app) register_scale_out_api_routers(app, supported_tasks)
if "transcription" in supported_tasks or "realtime" in supported_tasks: if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import ( from vllm.entrypoints.speech_to_text.factories import (
@@ -401,12 +392,6 @@ async def init_app_state(
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
) )
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
state.online_derenderer,
request_logger=request_logger,
)
if "generate" in supported_tasks: if "generate" in supported_tasks:
from vllm.entrypoints.generate.api_router import init_generate_state from vllm.entrypoints.generate.api_router import init_generate_state
@@ -415,6 +400,10 @@ async def init_app_state(
engine_client, state, args, request_logger, supported_tasks engine_client, state, args, request_logger, supported_tasks
) )
from vllm.entrypoints.scale_out.factories import init_scale_out_state
init_scale_out_state(state, args, engine_client, request_logger)
if "transcription" in supported_tasks or "realtime" in supported_tasks: if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
@@ -505,12 +494,10 @@ async def init_render_app_state(
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
) )
state.serving_render = ServingRender(
model_registry, from vllm.entrypoints.scale_out.factories import init_render_state
state.online_renderer,
state.online_derenderer, init_render_state(state, request_logger)
request_logger=request_logger,
)
state.vllm_config = vllm_config state.vllm_config = vllm_config
# Disable stats logging — there is no engine to poll. # Disable stats logging — there is no engine to poll.
@@ -0,0 +1,74 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
)
from .serving import ServingDerender
logger = init_logger(__name__)
router = APIRouter()
def derender(request: Request) -> ServingDerender | None:
return getattr(request.app.state, "serving_derender", None)
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=ChatCompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
handler = derender(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=CompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
handler = derender(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import cast
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
)
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import (
EngineInput,
MultiModalHashes,
MultiModalInput,
MultiModalPlaceholders,
)
from vllm.logger import init_logger
from vllm.renderers.online_derenderer import OnlineDerenderer
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
)
logger = init_logger(__name__)
class ServingDerender(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
online_derenderer: "OnlineDerenderer",
*,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=models.model_config,
request_logger=request_logger,
)
self.online_derenderer = online_derenderer
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
)
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
@staticmethod
def _extract_mm_features(
engine_input: EngineInput,
) -> MultiModalFeatures | None:
"""Extract multimodal metadata from a rendered engine prompt.
Returns ``None`` for text-only prompts.
"""
if engine_input.get("type") != "multimodal":
return None
# At this point engine_input is a MultiModalInput TypedDict.
mm_engine_input = cast(MultiModalInput, engine_input)
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
mm_placeholders = {
modality: [
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
]
for modality, ranges in raw_placeholders.items()
}
# Serialize tensor data per modality.
kwargs_data: dict[str, list[str | None]] | None = None
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
kwargs_data = {}
for modality, items in raw_mm_kwargs.items():
kwargs_data[modality] = [
encode_mm_kwargs_item(item) if item is not None else None
for item in items
]
return MultiModalFeatures(
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholders,
kwargs_data=kwargs_data,
)
+78
View File
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from typing import TYPE_CHECKING
from fastapi import FastAPI
from vllm.engine.protocol import EngineClient
from vllm.tasks import SupportedTask
if TYPE_CHECKING:
from starlette.datastructures import State
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
else:
RequestLogger = object
def init_render_state(
state: "State",
request_logger: RequestLogger | None,
):
from .derender.serving import ServingDerender
from .render.serving import ServingRender
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
)
state.serving_derender = ServingDerender(
state.openai_serving_models,
state.online_derenderer,
request_logger=request_logger,
)
def init_scale_out_state(
state: "State",
args: "Namespace",
engine_client: "EngineClient",
request_logger: RequestLogger | None,
):
init_render_state(state, request_logger)
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
state.serving_tokens = ServingTokens(
engine_client,
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_log_outputs=args.enable_log_outputs,
force_no_detokenize=args.tokens_only,
)
def register_scale_out_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
from .render.api_router import router as render_render
app.include_router(render_render)
from .derender.api_router import router as derender_render
app.include_router(derender_render)
if "generate" in supported_tasks:
from .token_in_token_out.api_router import (
attach_router as attach_disagg_router,
)
attach_disagg_router(app)
@@ -2,27 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
ChatCompletionRequest, from vllm.entrypoints.openai.completion.protocol import CompletionRequest
ChatCompletionResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionResponse,
)
from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest,
)
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.utils.api_utils import validate_json_request from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger from vllm.logger import init_logger
from ..token_in_token_out.protocol import GenerateRequest
from .serving import ServingRender
logger = init_logger(__name__) logger = init_logger(__name__)
router = APIRouter() router = APIRouter()
@@ -79,55 +70,3 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
return JSONResponse(content=result.model_dump(), status_code=result.error.code) return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=[item.model_dump() for item in result]) return JSONResponse(content=[item.model_dump() for item in result])
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=ChatCompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=CompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def attach_router(app: FastAPI) -> None:
app.include_router(router)
@@ -1,28 +1,16 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import cast from typing import cast
from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
ChatCompletionRequest, from vllm.entrypoints.openai.completion.protocol import CompletionRequest
ChatCompletionResponse, from vllm.entrypoints.openai.engine.protocol import ErrorResponse
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionResponse,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
)
from vllm.entrypoints.openai.models.serving import ( from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry, OpenAIModelRegistry,
OpenAIServingModels, OpenAIServingModels,
) )
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import encode_mm_kwargs_item
from vllm.entrypoints.serve.disagg.protocol import ( from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest, GenerateRequest,
MultiModalFeatures, MultiModalFeatures,
PlaceholderRangeInfo, PlaceholderRangeInfo,
@@ -41,7 +29,6 @@ from vllm.renderers.inputs.preprocess import (
extract_prompt_components, extract_prompt_components,
extract_prompt_len, extract_prompt_len,
) )
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer from vllm.renderers.online_renderer import OnlineRenderer
from vllm.utils import random_uuid from vllm.utils import random_uuid
@@ -53,7 +40,6 @@ class ServingRender(BaseServing):
self, self,
models: OpenAIServingModels | OpenAIModelRegistry, models: OpenAIServingModels | OpenAIModelRegistry,
online_renderer: "OnlineRenderer", online_renderer: "OnlineRenderer",
online_derenderer: "OnlineDerenderer",
*, *,
request_logger: RequestLogger | None = None, request_logger: RequestLogger | None = None,
) -> None: ) -> None:
@@ -64,7 +50,6 @@ class ServingRender(BaseServing):
) )
self.online_renderer = online_renderer self.online_renderer = online_renderer
self.online_derenderer = online_derenderer
self.default_sampling_params = ( self.default_sampling_params = (
online_renderer.model_config.get_diff_sampling_param() online_renderer.model_config.get_diff_sampling_param()
@@ -223,117 +208,6 @@ class ServingRender(BaseServing):
return generate_requests return generate_requests
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
)
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
@staticmethod @staticmethod
def _extract_mm_features( def _extract_mm_features(
engine_input: EngineInput, engine_input: EngineInput,
@@ -11,13 +11,6 @@ from fastapi.responses import JSONResponse, StreamingResponse
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
GenerateResponse,
)
from vllm.entrypoints.serve.disagg.serving import (
ServingTokens,
)
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import ( from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call, load_aware_call,
@@ -26,6 +19,12 @@ from vllm.entrypoints.serve.utils.api_utils import (
) )
from vllm.logger import init_logger from vllm.logger import init_logger
from .protocol import (
GenerateRequest,
GenerateResponse,
)
from .serving import ServingTokens
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -28,14 +28,6 @@ from vllm.entrypoints.openai.engine.protocol import (
) )
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
)
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput, mm_input from vllm.inputs import EngineInput, mm_input
@@ -51,6 +43,15 @@ from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils.collection_utils import as_list from vllm.utils.collection_utils import as_list
from .mm_serde import decode_mm_kwargs_item
from .protocol import (
GenerateRequest,
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
)
logger = init_logger(__name__) logger = init_logger(__name__)
+1 -1
View File
@@ -15,7 +15,7 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse, CompletionResponse,
) )
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.disagg.protocol import ( from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest, DerenderChatRequest,
DerenderCompletionRequest, DerenderCompletionRequest,
GenerateRequest, GenerateRequest,
+1 -1
View File
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.completion.protocol import (
) )
from vllm.entrypoints.openai.engine.protocol import ToolCall from vllm.entrypoints.openai.engine.protocol import ToolCall
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
from vllm.entrypoints.serve.disagg.protocol import GenerateResponse from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager from vllm.parser import Parser, ParserManager