[Frontend] Consolidate scale out entrypoints (#44512)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-29 03:18:53 -07:00
committed by GitHub
parent eddfd4cf21
commit 3483240b7e
45 changed files with 422 additions and 289 deletions
-15
View File
@@ -66,7 +66,6 @@ async def init_generate_state(
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
from vllm.entrypoints.serve.disagg.serving import ServingTokens
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
# Applied before any serving class is constructed so that each one picks
@@ -175,20 +174,6 @@ async def init_generate_state(
if "generate" in supported_tasks
else None
)
state.serving_tokens = (
ServingTokens(
engine_client,
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_log_outputs=args.enable_log_outputs,
force_no_detokenize=args.tokens_only,
)
if "generate" in supported_tasks
else None
)
from .generative_scoring.serving import ServingGenerativeScoring
+10 -23
View File
@@ -32,7 +32,6 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import (
@@ -208,12 +207,6 @@ def build_app(
register_generate_api_routers(app)
from vllm.entrypoints.serve.disagg.api_router import (
attach_router as attach_disagg_router,
)
attach_disagg_router(app)
from vllm.entrypoints.serve.elastic_ep.api_router import (
attach_router as elastic_ep_attach_router,
)
@@ -221,11 +214,9 @@ def build_app(
elastic_ep_attach_router(app)
if "generate" in supported_tasks or "render" in supported_tasks:
from vllm.entrypoints.serve.render.api_router import (
attach_router as attach_render_router,
)
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
attach_render_router(app)
register_scale_out_api_routers(app, supported_tasks)
if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import (
@@ -401,12 +392,6 @@ async def init_app_state(
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
state.online_derenderer,
request_logger=request_logger,
)
if "generate" in supported_tasks:
from vllm.entrypoints.generate.api_router import init_generate_state
@@ -415,6 +400,10 @@ async def init_app_state(
engine_client, state, args, request_logger, supported_tasks
)
from vllm.entrypoints.scale_out.factories import init_scale_out_state
init_scale_out_state(state, args, engine_client, request_logger)
if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
@@ -505,12 +494,10 @@ async def init_render_app_state(
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
state.serving_render = ServingRender(
model_registry,
state.online_renderer,
state.online_derenderer,
request_logger=request_logger,
)
from vllm.entrypoints.scale_out.factories import init_render_state
init_render_state(state, request_logger)
state.vllm_config = vllm_config
# Disable stats logging — there is no engine to poll.
@@ -0,0 +1,74 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
)
from .serving import ServingDerender
logger = init_logger(__name__)
router = APIRouter()
def derender(request: Request) -> ServingDerender | None:
return getattr(request.app.state, "serving_derender", None)
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=ChatCompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
handler = derender(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=CompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
handler = derender(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import cast
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
)
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import (
EngineInput,
MultiModalHashes,
MultiModalInput,
MultiModalPlaceholders,
)
from vllm.logger import init_logger
from vllm.renderers.online_derenderer import OnlineDerenderer
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
)
logger = init_logger(__name__)
class ServingDerender(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
online_derenderer: "OnlineDerenderer",
*,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=models.model_config,
request_logger=request_logger,
)
self.online_derenderer = online_derenderer
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
)
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
@staticmethod
def _extract_mm_features(
engine_input: EngineInput,
) -> MultiModalFeatures | None:
"""Extract multimodal metadata from a rendered engine prompt.
Returns ``None`` for text-only prompts.
"""
if engine_input.get("type") != "multimodal":
return None
# At this point engine_input is a MultiModalInput TypedDict.
mm_engine_input = cast(MultiModalInput, engine_input)
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
mm_placeholders = {
modality: [
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
]
for modality, ranges in raw_placeholders.items()
}
# Serialize tensor data per modality.
kwargs_data: dict[str, list[str | None]] | None = None
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
kwargs_data = {}
for modality, items in raw_mm_kwargs.items():
kwargs_data[modality] = [
encode_mm_kwargs_item(item) if item is not None else None
for item in items
]
return MultiModalFeatures(
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholders,
kwargs_data=kwargs_data,
)
+78
View File
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from typing import TYPE_CHECKING
from fastapi import FastAPI
from vllm.engine.protocol import EngineClient
from vllm.tasks import SupportedTask
if TYPE_CHECKING:
from starlette.datastructures import State
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
else:
RequestLogger = object
def init_render_state(
state: "State",
request_logger: RequestLogger | None,
):
from .derender.serving import ServingDerender
from .render.serving import ServingRender
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
)
state.serving_derender = ServingDerender(
state.openai_serving_models,
state.online_derenderer,
request_logger=request_logger,
)
def init_scale_out_state(
state: "State",
args: "Namespace",
engine_client: "EngineClient",
request_logger: RequestLogger | None,
):
init_render_state(state, request_logger)
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
state.serving_tokens = ServingTokens(
engine_client,
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_log_outputs=args.enable_log_outputs,
force_no_detokenize=args.tokens_only,
)
def register_scale_out_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
from .render.api_router import router as render_render
app.include_router(render_render)
from .derender.api_router import router as derender_render
app.include_router(derender_render)
if "generate" in supported_tasks:
from .token_in_token_out.api_router import (
attach_router as attach_disagg_router,
)
attach_disagg_router(app)
@@ -2,27 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionResponse,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest,
)
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import GenerateRequest
from .serving import ServingRender
logger = init_logger(__name__)
router = APIRouter()
@@ -79,55 +70,3 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=[item.model_dump() for item in result])
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=ChatCompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=CompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def attach_router(app: FastAPI) -> None:
app.include_router(router)
@@ -1,28 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import cast
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionResponse,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import encode_mm_kwargs_item
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
@@ -41,7 +29,6 @@ from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
extract_prompt_len,
)
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.utils import random_uuid
@@ -53,7 +40,6 @@ class ServingRender(BaseServing):
self,
models: OpenAIServingModels | OpenAIModelRegistry,
online_renderer: "OnlineRenderer",
online_derenderer: "OnlineDerenderer",
*,
request_logger: RequestLogger | None = None,
) -> None:
@@ -64,7 +50,6 @@ class ServingRender(BaseServing):
)
self.online_renderer = online_renderer
self.online_derenderer = online_derenderer
self.default_sampling_params = (
online_renderer.model_config.get_diff_sampling_param()
@@ -223,117 +208,6 @@ class ServingRender(BaseServing):
return generate_requests
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
)
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
@staticmethod
def _extract_mm_features(
engine_input: EngineInput,
@@ -11,13 +11,6 @@ from fastapi.responses import JSONResponse, StreamingResponse
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
GenerateResponse,
)
from vllm.entrypoints.serve.disagg.serving import (
ServingTokens,
)
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
@@ -26,6 +19,12 @@ from vllm.entrypoints.serve.utils.api_utils import (
)
from vllm.logger import init_logger
from .protocol import (
GenerateRequest,
GenerateResponse,
)
from .serving import ServingTokens
logger = init_logger(__name__)
@@ -28,14 +28,6 @@ from vllm.entrypoints.openai.engine.protocol import (
)
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
)
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput, mm_input
@@ -51,6 +43,15 @@ from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils.collection_utils import as_list
from .mm_serde import decode_mm_kwargs_item
from .protocol import (
GenerateRequest,
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
)
logger = init_logger(__name__)
+1 -1
View File
@@ -15,7 +15,7 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.disagg.protocol import (
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest,