[Frontend] Consolidate scale out entrypoints (#44512)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
)
|
||||
from .serving import ServingDerender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def derender(request: Request) -> ServingDerender | None:
|
||||
return getattr(request.app.state, "serving_derender", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=ChatCompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Derender API"
|
||||
)
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=CompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Derender API")
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from typing import cast
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import (
|
||||
EngineInput,
|
||||
MultiModalHashes,
|
||||
MultiModalInput,
|
||||
MultiModalPlaceholders,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
|
||||
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingDerender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
online_derenderer: "OnlineDerenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=models.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.online_derenderer = online_derenderer
|
||||
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: DerenderChatRequest,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||
|
||||
Non-streaming only: expects the complete GenerateResponse with all
|
||||
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
|
||||
|
||||
When ``request.chat_request`` is provided, the parser splits the
|
||||
output into (reasoning, content, tool_calls). Otherwise falls
|
||||
back to plain detokenization.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
try:
|
||||
choices = await self.online_derenderer.derender_chat(
|
||||
request.generate_response, request.chat_request
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
prompt_tokens = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||
)
|
||||
gen = request.generate_response
|
||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||
gen.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
completion_tokens,
|
||||
)
|
||||
return ChatCompletionResponse(
|
||||
id=gen.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
self,
|
||||
request: DerenderCompletionRequest,
|
||||
) -> CompletionResponse | ErrorResponse:
|
||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||
|
||||
Non-streaming only. Mirrors the multi-prompt completions case: one
|
||||
GenerateResponse per prompt, parallel to the list[GenerateRequest]
|
||||
from /v1/completions/render.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
(
|
||||
choices,
|
||||
total_prompt_tokens,
|
||||
total_completion_tokens,
|
||||
) = await self.online_derenderer.derender_completion(
|
||||
request.generate_responses, request.prompt_tokens
|
||||
)
|
||||
|
||||
if not request.generate_responses:
|
||||
return self.create_error_response("generate_responses must not be empty")
|
||||
|
||||
first = request.generate_responses[0]
|
||||
kv_params = first.kv_transfer_params
|
||||
if any(
|
||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||
):
|
||||
logger.warning(
|
||||
"derender_completion: kv_transfer_params differ across responses; "
|
||||
"setting to None on the aggregated response"
|
||||
)
|
||||
kv_params = None
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion request_id=%s model=%s choices=%d"
|
||||
" completion_tokens=%d",
|
||||
first.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
total_completion_tokens,
|
||||
)
|
||||
return CompletionResponse(
|
||||
id=first.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
) -> MultiModalFeatures | None:
|
||||
"""Extract multimodal metadata from a rendered engine prompt.
|
||||
|
||||
Returns ``None`` for text-only prompts.
|
||||
"""
|
||||
if engine_input.get("type") != "multimodal":
|
||||
return None
|
||||
|
||||
# At this point engine_input is a MultiModalInput TypedDict.
|
||||
mm_engine_input = cast(MultiModalInput, engine_input)
|
||||
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
|
||||
raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
|
||||
|
||||
mm_placeholders = {
|
||||
modality: [
|
||||
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in raw_placeholders.items()
|
||||
}
|
||||
|
||||
# Serialize tensor data per modality.
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
|
||||
kwargs_data = {}
|
||||
for modality, items in raw_mm_kwargs.items():
|
||||
kwargs_data[modality] = [
|
||||
encode_mm_kwargs_item(item) if item is not None else None
|
||||
for item in items
|
||||
]
|
||||
|
||||
return MultiModalFeatures(
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
kwargs_data=kwargs_data,
|
||||
)
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
else:
|
||||
RequestLogger = object
|
||||
|
||||
|
||||
def init_render_state(
|
||||
state: "State",
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
from .derender.serving import ServingDerender
|
||||
from .render.serving import ServingRender
|
||||
|
||||
state.serving_render = ServingRender(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
state.serving_derender = ServingDerender(
|
||||
state.openai_serving_models,
|
||||
state.online_derenderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
|
||||
def init_scale_out_state(
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
engine_client: "EngineClient",
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
init_render_state(state, request_logger)
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
|
||||
state.serving_tokens = ServingTokens(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
force_no_detokenize=args.tokens_only,
|
||||
)
|
||||
|
||||
|
||||
def register_scale_out_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
from .render.api_router import router as render_render
|
||||
|
||||
app.include_router(render_render)
|
||||
|
||||
from .derender.api_router import router as derender_render
|
||||
|
||||
app.include_router(derender_render)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from .token_in_token_out.api_router import (
|
||||
attach_router as attach_disagg_router,
|
||||
)
|
||||
|
||||
attach_disagg_router(app)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import GenerateRequest
|
||||
from .serving import ServingRender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Render API"
|
||||
)
|
||||
|
||||
result = await handler.render_chat_request(request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=list[GenerateRequest],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_completion(request: CompletionRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Render API")
|
||||
|
||||
result = await handler.render_completion_request(request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=[item.model_dump() for item in result])
|
||||
@@ -0,0 +1,248 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import cast
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import encode_mm_kwargs_item
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import (
|
||||
EngineInput,
|
||||
MultiModalHashes,
|
||||
MultiModalInput,
|
||||
MultiModalPlaceholders,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
)
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingRender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
online_renderer: "OnlineRenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=online_renderer.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.online_renderer = online_renderer
|
||||
|
||||
self.default_sampling_params = (
|
||||
online_renderer.model_config.get_diff_sampling_param()
|
||||
)
|
||||
mc = online_renderer.model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
async def render_chat_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
"""Validate the model and preprocess a chat completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
GPU-less render server and delegated to by OpenAIServingChat.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
if request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
_, engine_inputs = result
|
||||
|
||||
if len(engine_inputs) != 1:
|
||||
return self.create_error_response(
|
||||
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
|
||||
)
|
||||
|
||||
engine_input = engine_inputs[0]
|
||||
|
||||
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_completion_tokens
|
||||
if request.max_completion_tokens is not None
|
||||
else request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment]
|
||||
"assistant_tokens_mask"
|
||||
)
|
||||
if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len(
|
||||
token_ids
|
||||
):
|
||||
logger.warning(
|
||||
"assistant_tokens_mask length (%d) != token_ids length (%d); "
|
||||
"this can happen with multimodal inputs where "
|
||||
"placeholder expansion changes the token count. "
|
||||
"The mask may be positionally misaligned.",
|
||||
len(assistant_tokens_mask),
|
||||
len(token_ids),
|
||||
)
|
||||
if len(assistant_tokens_mask) < len(token_ids):
|
||||
assistant_tokens_mask.extend(
|
||||
[0] * (len(token_ids) - len(assistant_tokens_mask))
|
||||
)
|
||||
else:
|
||||
assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)]
|
||||
|
||||
request_id = f"chatcmpl-{random_uuid()}"
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
assistant_tokens_mask=assistant_tokens_mask,
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
async def render_completion_request(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
) -> list[GenerateRequest] | ErrorResponse:
|
||||
"""Validate the model and preprocess a completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
GPU-less render server and delegated to by OpenAIServingCompletion.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
result = await self.online_renderer.render_completion(
|
||||
request, skip_mm_cache=True
|
||||
)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
generate_requests: list[GenerateRequest] = []
|
||||
for engine_input in result:
|
||||
prompt_components = extract_prompt_components(
|
||||
self.model_config, engine_input
|
||||
)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
)
|
||||
|
||||
request_id = f"cmpl-{random_uuid()}"
|
||||
|
||||
generate_requests.append(
|
||||
GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
)
|
||||
|
||||
return generate_requests
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
) -> MultiModalFeatures | None:
|
||||
"""Extract multimodal metadata from a rendered engine prompt.
|
||||
|
||||
Returns ``None`` for text-only prompts.
|
||||
"""
|
||||
if engine_input.get("type") != "multimodal":
|
||||
return None
|
||||
|
||||
# At this point engine_input is a MultiModalInput TypedDict.
|
||||
mm_engine_input = cast(MultiModalInput, engine_input)
|
||||
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
|
||||
raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
|
||||
|
||||
mm_placeholders = {
|
||||
modality: [
|
||||
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in raw_placeholders.items()
|
||||
}
|
||||
|
||||
# Serialize tensor data per modality.
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
|
||||
kwargs_data = {}
|
||||
for modality, items in raw_mm_kwargs.items():
|
||||
kwargs_data[modality] = [
|
||||
encode_mm_kwargs_item(item) if item is not None else None
|
||||
for item in items
|
||||
]
|
||||
|
||||
return MultiModalFeatures(
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
kwargs_data=kwargs_data,
|
||||
)
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
)
|
||||
from .serving import ServingTokens
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def tokenization(request: Request) -> ServingTokenization:
|
||||
return request.app.state.serving_tokenization
|
||||
|
||||
|
||||
def generate_tokens(request: Request) -> ServingTokens | None:
|
||||
return request.app.state.serving_tokens
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/inference/v1/generate",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def generate(request: GenerateRequest, raw_request: Request):
|
||||
handler = generate_tokens(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support generate tokens API")
|
||||
|
||||
generator = await handler.serve_tokens(request, raw_request)
|
||||
|
||||
if isinstance(generator, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=generator.model_dump(), status_code=generator.error.code
|
||||
)
|
||||
|
||||
elif isinstance(generator, GenerateResponse):
|
||||
return JSONResponse(content=generator.model_dump())
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
if getattr(app.state.args, "tokens_only", False):
|
||||
|
||||
@router.post("/abort_requests")
|
||||
async def abort_requests(raw_request: Request):
|
||||
"""
|
||||
Abort one or more requests. To be used in a
|
||||
Disaggregated Everything setup.
|
||||
"""
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
request_ids = body.get("request_ids")
|
||||
if request_ids is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Missing 'request_ids' in request body",
|
||||
)
|
||||
# Abort requests in background
|
||||
asyncio.create_task(engine_client(raw_request).abort(request_ids))
|
||||
return Response(status_code=200)
|
||||
|
||||
app.include_router(router)
|
||||
@@ -0,0 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Encode/decode utilities for multimodal tensors and field metadata
|
||||
over JSON/HTTP, used by the disaggregated generate endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pybase64
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
_encoder = MsgpackEncoder(size_threshold=2**62) # force all tensors inline
|
||||
_decoder = MsgpackDecoder(t=MultiModalKwargsItem)
|
||||
|
||||
|
||||
def encode_mm_kwargs_item(item: MultiModalKwargsItem) -> str:
|
||||
"""Serialize a MultiModalKwargsItem to a base64 string."""
|
||||
bufs = _encoder.encode(item)
|
||||
assert len(bufs) == 1, "All tensors should be inline"
|
||||
return pybase64.b64encode(bufs[0]).decode("ascii")
|
||||
|
||||
|
||||
def decode_mm_kwargs_item(data: str) -> MultiModalKwargsItem:
|
||||
"""Deserialize a base64 string back to a MultiModalKwargsItem."""
|
||||
raw = pybase64.b64decode(data)
|
||||
return _decoder.decode(raw)
|
||||
@@ -0,0 +1,298 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
Field,
|
||||
PrivateAttr,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
####### Tokens IN <> Tokens OUT #######
|
||||
|
||||
|
||||
class PlaceholderRangeInfo(BaseModel):
|
||||
"""Serializable placeholder location for a single multi-modal item."""
|
||||
|
||||
offset: int
|
||||
"""Start index of the placeholder tokens in the prompt."""
|
||||
|
||||
length: int
|
||||
"""Number of placeholder tokens."""
|
||||
|
||||
# TODO: add ``is_embed: list[bool] | None`` once the /generate side
|
||||
# consumes features — some models (e.g. Qwen-VL) use sparse
|
||||
# placeholder masks that cannot be recomputed from offset+length alone.
|
||||
|
||||
|
||||
class MultiModalFeatures(BaseModel):
|
||||
"""Lightweight multimodal metadata produced by the render step.
|
||||
|
||||
Carries hashes (for cache lookup / identification) and placeholder
|
||||
positions so the downstream `/generate` service knows *where* in
|
||||
the token sequence each multimodal item lives.
|
||||
"""
|
||||
|
||||
mm_hashes: dict[str, list[str]]
|
||||
"""Per-modality item hashes, e.g. `{"image": ["abc", "def"]}`."""
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
|
||||
"""Per-modality placeholder ranges in the token sequence."""
|
||||
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
"""Per-modality serialized tensor data.
|
||||
|
||||
Each value is a list parallel to ``mm_hashes[modality]``. A ``str``
|
||||
entry is a base64-encoded ``MultiModalKwargsItem``; ``None`` means
|
||||
the item should be resolved from cache. The entire field is
|
||||
``None`` for metadata-only (cache-hit) responses.
|
||||
"""
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
token_ids: list[int] = Field(min_length=1)
|
||||
"""The token ids to generate text from."""
|
||||
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
"""Per-token mask (1 = assistant-generated, 0 = not).
|
||||
|
||||
Only populated when the render request sets ``return_assistant_tokens_mask=True``
|
||||
and the chat template supports ``{% generation %}``.
|
||||
``None`` when the mask was not requested or could not be computed.
|
||||
"""
|
||||
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
def validate_token_ids(cls, v: list[int]) -> list[int]:
|
||||
if any(t < 0 for t in v):
|
||||
raise ValueError("token_ids must not contain negative values")
|
||||
return v
|
||||
|
||||
token_offsets: list[tuple[int, int]] | None = None
|
||||
"""Char-level (start, end) offsets per token, relative to the
|
||||
tokenized source string. Present only when the request set
|
||||
`return_token_offsets=True` and the renderer was able to compute
|
||||
them (Fast tokenizer, text input, no multimodal data). List length
|
||||
equals `token_ids` length when present. None otherwise."""
|
||||
|
||||
features: MultiModalFeatures | None = None
|
||||
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
|
||||
|
||||
sampling_params: SamplingParams
|
||||
"""The sampling parameters for the model."""
|
||||
|
||||
model: str | None = None
|
||||
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(
|
||||
default=0,
|
||||
ge=-(2**63),
|
||||
le=2**63 - 1,
|
||||
description=(
|
||||
"The priority of the request (lower means earlier handling; "
|
||||
"default: 0). Any priority other than 0 will raise an error "
|
||||
"if the served model does not use priority scheduling."
|
||||
),
|
||||
)
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
# Tracks which keys the caller explicitly set inside ``sampling_params``
|
||||
# when the request was parsed from a JSON body. Lets the server tell
|
||||
# "client said max_tokens=16" from "client said nothing → dataclass
|
||||
# default 16" so it can apply server-side defaulting only in the latter
|
||||
# case. ``None`` means the request was constructed with a pre-built
|
||||
# ``SamplingParams`` instance (e.g. from internal callers that have
|
||||
# already resolved values), in which case all fields are considered set.
|
||||
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
|
||||
|
||||
@model_validator(mode="wrap")
|
||||
@classmethod
|
||||
def _capture_sampling_params_provided_keys(cls, data: Any, handler):
|
||||
provided: set[str] | None = None
|
||||
if isinstance(data, dict):
|
||||
sp = data.get("sampling_params")
|
||||
if isinstance(sp, dict):
|
||||
provided = set(sp.keys())
|
||||
instance = handler(data)
|
||||
instance._sampling_params_provided_keys = provided
|
||||
return instance
|
||||
|
||||
def is_sampling_param_provided(self, name: str) -> bool:
|
||||
"""Whether the caller explicitly set ``sampling_params.<name>``.
|
||||
|
||||
For requests parsed from a JSON body, this reflects the raw input
|
||||
dict. For requests constructed with a pre-built ``SamplingParams``
|
||||
instance, all fields are considered provided so server-side defaults
|
||||
do not clobber values already resolved upstream.
|
||||
"""
|
||||
if self._sampling_params_provided_keys is None:
|
||||
return True
|
||||
return name in self._sampling_params_provided_keys
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=None,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
|
||||
class GenerateResponseChoice(BaseModel):
|
||||
index: int
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
# np.load(io.BytesIO(base64.b64decode(s)))
|
||||
# ``None`` if (a) the request was aborted before any forward pass,
|
||||
# or (b) ``enable_return_routed_experts`` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
|
||||
|
||||
class GenerateResponseStreamChoice(BaseModel):
|
||||
index: int
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
finish_reason: str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
routed_experts: str | None = None
|
||||
|
||||
|
||||
class GenerateStreamResponse(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
choices: list[GenerateResponseStreamChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
|
||||
|
||||
class GenerateResponse(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
choices: list[GenerateResponseChoice]
|
||||
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
|
||||
####### Derender (postprocessing) #######
|
||||
|
||||
|
||||
class DerenderChatRequest(BaseModel):
|
||||
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
|
||||
|
||||
Wraps a complete GenerateResponse and caller-supplied metadata needed to
|
||||
produce a fully-formed ChatCompletionResponse without a GPU.
|
||||
|
||||
Streaming derender would require a separate endpoint design with
|
||||
incremental token delivery, ``OutputProcessor``-based detokenization,
|
||||
and ``parser.parse_delta()`` instead of ``parser.parse()``.
|
||||
"""
|
||||
|
||||
model: str
|
||||
generate_response: GenerateResponse
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage; defaults to 0 if omitted.
|
||||
|
||||
GenerateResponse carries only output tokens; the caller already has
|
||||
len(GenerateRequest.token_ids) from the render step.
|
||||
"""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
"""The original (post-adjust_request) ChatCompletionRequest from /render.
|
||||
|
||||
Required by the parsing so that tool/reasoning parsers can receive the full
|
||||
request context they expect (request.tools, request.tool_choice,
|
||||
request._grammar_from_tool_parser, etc.).
|
||||
"""
|
||||
|
||||
|
||||
class DerenderCompletionRequest(BaseModel):
|
||||
"""Request for the /v1/completions/derender endpoint (non-streaming).
|
||||
|
||||
Parallel to DerenderChatRequest but handles the multi-prompt completions
|
||||
case: one GenerateResponse per prompt, mirroring the list[GenerateRequest]
|
||||
returned by /v1/completions/render.
|
||||
"""
|
||||
|
||||
model: str
|
||||
generate_responses: list[GenerateResponse]
|
||||
prompt_tokens: list[int] | None = None
|
||||
"""One prompt token count per response; each defaults to 0 if omitted.
|
||||
|
||||
If provided, len(prompt_tokens) must equal len(generate_responses).
|
||||
"""
|
||||
|
||||
completion_request: CompletionRequest | None = None
|
||||
"""The original (post-adjust_request) CompletionRequest from /render.
|
||||
|
||||
Mirrors chat_request on DerenderChatRequest. Required by the parsing
|
||||
so parsers receive the full request context.
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_prompt_tokens_length(self) -> "DerenderCompletionRequest":
|
||||
if self.prompt_tokens is not None and len(self.prompt_tokens) != len(
|
||||
self.generate_responses
|
||||
):
|
||||
raise ValueError(
|
||||
f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
|
||||
f"generate_responses length ({len(self.generate_responses)})"
|
||||
)
|
||||
return self
|
||||
@@ -0,0 +1,504 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProb,
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionLogProbsContent,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.utils.collection_utils import as_list
|
||||
|
||||
from .mm_serde import decode_mm_kwargs_item
|
||||
from .protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingTokens(OpenAIServing):
|
||||
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
online_renderer: OnlineRenderer,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
force_no_detokenize: bool = False,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
self.online_renderer = online_renderer
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_log_outputs = enable_log_outputs
|
||||
self.force_no_detokenize = force_no_detokenize
|
||||
if force_no_detokenize:
|
||||
logger.info(
|
||||
"Tokens-only mode is enabled, skipping detokenization "
|
||||
"step for incoming requests."
|
||||
)
|
||||
|
||||
# Mirrors ``OpenAIServingChat`` so we can apply server-side
|
||||
# ``max_tokens`` defaulting when the client omits it. Without this,
|
||||
# ``SamplingParams.max_tokens`` falls back to its dataclass default
|
||||
# of 16 and silently truncates every generation.
|
||||
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
||||
mc = self.model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
async def serve_tokens(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> GenerateResponse | ErrorResponse | AsyncGenerator[str, None]:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
lora_request = None
|
||||
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
|
||||
|
||||
model_name = self.models.model_name(lora_request)
|
||||
|
||||
request_id = (
|
||||
f"generate-tokens-{self._base_request_id(raw_request, request.request_id)}"
|
||||
)
|
||||
|
||||
request_metadata = RequestResponseMetadata(request_id=request_id)
|
||||
if raw_request:
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
sampling_params = request.sampling_params
|
||||
max_num_seqs = self.engine_client.vllm_config.scheduler_config.max_num_seqs
|
||||
if sampling_params.n > max_num_seqs:
|
||||
return self.create_error_response(
|
||||
f"sampling_params.n must be at most the server's max_num_seqs "
|
||||
f"({max_num_seqs}), got {sampling_params.n}."
|
||||
)
|
||||
try:
|
||||
msgspec.msgpack.encode(sampling_params)
|
||||
except (OverflowError, TypeError, ValueError) as e:
|
||||
return self.create_error_response(e)
|
||||
|
||||
engine_input: EngineInput
|
||||
if features := request.features:
|
||||
# Convert PlaceholderRangeInfo → PlaceholderRange per modality.
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {
|
||||
modality: [
|
||||
PlaceholderRange(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in features.mm_placeholders.items()
|
||||
}
|
||||
|
||||
# Deserialize tensor data when present; None → cache hit.
|
||||
mm_kwargs: dict[str, list[MultiModalKwargsItem | None]] = {}
|
||||
if features.kwargs_data is not None:
|
||||
for modality, items in features.kwargs_data.items():
|
||||
mm_kwargs[modality] = [
|
||||
decode_mm_kwargs_item(item) if item is not None else None
|
||||
for item in items
|
||||
]
|
||||
else:
|
||||
for modality, hashes in features.mm_hashes.items():
|
||||
mm_kwargs[modality] = [None] * len(hashes)
|
||||
|
||||
engine_input = mm_input(
|
||||
prompt_token_ids=request.token_ids,
|
||||
mm_kwargs=MultiModalKwargsItems(mm_kwargs),
|
||||
mm_hashes=features.mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
cache_salt=request.cache_salt,
|
||||
)
|
||||
else:
|
||||
(engine_input,) = await self.online_renderer.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.token_ids,
|
||||
prompt_embeds=None,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
||||
|
||||
# Apply server-side ``max_tokens`` defaulting when the client did
|
||||
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
|
||||
# defaults ``max_tokens`` to 16, which would otherwise silently cap
|
||||
# every generation that omits the field.
|
||||
if not request.is_sampling_param_provided("max_tokens"):
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
max_model_len=self.model_config.max_model_len,
|
||||
max_tokens=None,
|
||||
input_length=self._extract_prompt_len(engine_input),
|
||||
default_sampling_params=self.default_sampling_params,
|
||||
override_max_tokens=self.override_max_tokens,
|
||||
)
|
||||
|
||||
if self.force_no_detokenize:
|
||||
sampling_params.detokenize = False
|
||||
if request.stream:
|
||||
sampling_params.output_kind = RequestOutputKind.DELTA
|
||||
|
||||
self._log_inputs(
|
||||
request_id,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
# Extract data_parallel_rank from header (router can inject it)
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
|
||||
result_generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
|
||||
assert result_generator is not None
|
||||
|
||||
if request.stream:
|
||||
return self.serve_tokens_stream_generator(
|
||||
request,
|
||||
result_generator,
|
||||
request_id,
|
||||
model_name,
|
||||
request_metadata,
|
||||
)
|
||||
|
||||
return await self.serve_tokens_full_generator(
|
||||
request, result_generator, request_id, model_name, request_metadata
|
||||
)
|
||||
|
||||
async def serve_tokens_full_generator(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
result_generator: AsyncGenerator[RequestOutput, None],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
) -> ErrorResponse | GenerateResponse:
|
||||
created_time = int(time.time())
|
||||
final_res: RequestOutput | None = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
final_res = res
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
|
||||
assert final_res is not None
|
||||
|
||||
choices: list[GenerateResponseChoice] = []
|
||||
num_generated_tokens = 0
|
||||
for output in final_res.outputs:
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
# This is top_logprobs in completions API
|
||||
if sampling_params.logprobs is not None:
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
# Encode routed_experts for transport. JSON can't carry raw
|
||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
||||
# This is the only base64 hop in the pipeline -- the
|
||||
# engine<->API-server link is binary msgpack + zmq.
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
|
||||
choice_data = GenerateResponseChoice(
|
||||
index=output.index,
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason if output.finish_reason else "stop",
|
||||
token_ids=as_list(output.token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
)
|
||||
|
||||
choices.append(choice_data)
|
||||
num_generated_tokens += len(output.token_ids)
|
||||
|
||||
assert final_res.prompt_token_ids is not None
|
||||
num_prompt_tokens = len(final_res.prompt_token_ids)
|
||||
if final_res.encoder_prompt_token_ids is not None:
|
||||
num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
)
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and final_res.num_cached_tokens is not None
|
||||
):
|
||||
# This info is not available at the /coordinator level
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=final_res.num_cached_tokens
|
||||
)
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
response = GenerateResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
)
|
||||
|
||||
# Log complete response if output logging is enabled
|
||||
if self.enable_log_outputs and self.request_logger:
|
||||
for choice in choices:
|
||||
# Get the corresponding output token IDs
|
||||
output_token_ids = None
|
||||
if choice.index < len(final_res.outputs):
|
||||
output_token_ids = final_res.outputs[choice.index].token_ids
|
||||
|
||||
if output_token_ids:
|
||||
# Log token_ids only.
|
||||
self.request_logger.log_outputs(
|
||||
request_id=request_id,
|
||||
outputs="",
|
||||
output_token_ids=output_token_ids,
|
||||
finish_reason=choice.finish_reason,
|
||||
is_streaming=False,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
async def serve_tokens_stream_generator(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
result_generator: AsyncGenerator[RequestOutput, None],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
num_prompt_tokens = 0
|
||||
num_generated_tokens: list[int] = []
|
||||
first_iteration = True
|
||||
num_cached_tokens = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
|
||||
include_usage, include_continuous_usage = should_include_usage(
|
||||
request.stream_options, False
|
||||
)
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
if first_iteration:
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
if res.encoder_prompt_token_ids is not None:
|
||||
num_prompt_tokens += len(res.encoder_prompt_token_ids)
|
||||
num_cached_tokens = res.num_cached_tokens
|
||||
num_generated_tokens = [0] * len(res.outputs)
|
||||
first_iteration = False
|
||||
|
||||
for output in res.outputs:
|
||||
i = output.index
|
||||
delta_token_ids = output.token_ids
|
||||
num_generated_tokens[i] += len(delta_token_ids)
|
||||
|
||||
finish_reason = output.finish_reason
|
||||
self._raise_if_error(finish_reason, request_id)
|
||||
|
||||
if not delta_token_ids:
|
||||
continue
|
||||
|
||||
if sampling_params.logprobs is not None:
|
||||
out_logprobs = output.logprobs
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
|
||||
"ascii"
|
||||
)
|
||||
|
||||
chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
GenerateResponseStreamChoice(
|
||||
index=i,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
token_ids=as_list(delta_token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
)
|
||||
],
|
||||
)
|
||||
if include_continuous_usage:
|
||||
chunk.usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=num_generated_tokens[i],
|
||||
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
|
||||
)
|
||||
|
||||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||||
|
||||
total_completion_tokens = sum(num_generated_tokens)
|
||||
final_usage_info = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=num_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
final_chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
)
|
||||
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
request_metadata.final_usage_info = final_usage_info
|
||||
|
||||
except GenerationError as e:
|
||||
yield (
|
||||
f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error in token generation stream.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
def _create_tokens_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Create OpenAI-style logprobs."""
|
||||
logprobs_content: list[ChatCompletionLogProbsContent] = []
|
||||
|
||||
for i, token_id in enumerate(token_ids):
|
||||
token = f"token_id:{token_id}"
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
logprobs_content.append(
|
||||
ChatCompletionLogProbsContent(
|
||||
token=token,
|
||||
)
|
||||
)
|
||||
else:
|
||||
step_token = step_top_logprobs[token_id]
|
||||
|
||||
logprobs_content.append(
|
||||
ChatCompletionLogProbsContent(
|
||||
token=token,
|
||||
logprob=max(step_token.logprob, -9999.0),
|
||||
top_logprobs=[
|
||||
ChatCompletionLogProb(
|
||||
token=f"token_id:{token_id}",
|
||||
logprob=max(logprob.logprob, -9999.0),
|
||||
)
|
||||
for i, (token_id, logprob) in enumerate(
|
||||
step_top_logprobs.items()
|
||||
)
|
||||
if num_output_top_logprobs is not None
|
||||
and i < max(num_output_top_logprobs, 1)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
return ChatCompletionLogProbs(content=logprobs_content)
|
||||
Reference in New Issue
Block a user