[Frontend] Consolidate scale out entrypoints (#44512)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -845,10 +845,12 @@ steps:
|
|||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/
|
- vllm/
|
||||||
- tests/entrypoints/serve
|
- tests/entrypoints/serve
|
||||||
|
- tests/entrypoints/scale_out
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
|
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
|
||||||
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
|
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
|
||||||
|
- pytest -v -s entrypoints/scale_out
|
||||||
|
|
||||||
- label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
|
- label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -2559,10 +2561,12 @@ steps:
|
|||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/
|
- vllm/
|
||||||
- tests/entrypoints/serve
|
- tests/entrypoints/serve
|
||||||
|
- tests/entrypoints/scale_out
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
|
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
|
||||||
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
|
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
|
||||||
|
- pytest -v -s entrypoints/scale_out
|
||||||
|
|
||||||
- label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
|
- label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
|
|||||||
@@ -40,10 +40,12 @@ steps:
|
|||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/
|
- vllm/
|
||||||
- tests/entrypoints/serve
|
- tests/entrypoints/serve
|
||||||
|
- tests/entrypoints/scale_out
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
|
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
|
||||||
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
|
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
|
||||||
|
- pytest -v -s entrypoints/scale_out
|
||||||
mirror:
|
mirror:
|
||||||
amd:
|
amd:
|
||||||
device: mi325_1
|
device: mi325_1
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ steps:
|
|||||||
- vllm/v1/engine/
|
- vllm/v1/engine/
|
||||||
- tests/utils.py
|
- tests/utils.py
|
||||||
# - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
|
# - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
|
||||||
- tests/entrypoints/serve/disagg/test_serving_tokens.py
|
- tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
|
||||||
- tests/entrypoints/serve/instrumentator/test_basic.py
|
- tests/entrypoints/serve/instrumentator/test_basic.py
|
||||||
- tests/entrypoints/serve/instrumentator/test_metrics.py
|
- tests/entrypoints/serve/instrumentator/test_metrics.py
|
||||||
# - tests/entrypoints/serve/dev/test_sleep.py
|
# - tests/entrypoints/serve/dev/test_sleep.py
|
||||||
@@ -55,7 +55,7 @@ steps:
|
|||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
# - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
|
# - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
|
||||||
- pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
|
- pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
|
||||||
- pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
|
- pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
|
||||||
- pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
|
- pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
|
||||||
# - pytest -v -s entrypoints/serve/dev/test_sleep.py
|
# - pytest -v -s entrypoints/serve/dev/test_sleep.py
|
||||||
|
|
||||||
|
|||||||
@@ -13,5 +13,6 @@ vLLM's examples are organized into the following categories:
|
|||||||
- **[`rl/`](../../examples/rl)** – Reinforcement learning examples.
|
- **[`rl/`](../../examples/rl)** – Reinforcement learning examples.
|
||||||
- **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production.
|
- **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production.
|
||||||
- **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray.
|
- **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray.
|
||||||
- **[`disaggregated/`](../../examples/disaggregated)** – Examples for disaggregated serving (separate prefill and decode), including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
|
- **[`disaggregated/`](../../examples/disaggregated)** – Examples for Disaggregated P/D (Prefill/Decoding) inference, including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
|
||||||
|
- **[`scale_out/`](../../examples/scale_out)** – Examples for Token In <> Token Out API Server.
|
||||||
- **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses).
|
- **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses).
|
||||||
|
|||||||
@@ -119,9 +119,9 @@ For further details on profiling vLLM, please refer to [this page](../../contrib
|
|||||||
- `/ping` - SageMaker health check
|
- `/ping` - SageMaker health check
|
||||||
- `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
|
- `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
|
||||||
|
|
||||||
## Disaggregated Everything
|
## Scale-Out APIs
|
||||||
|
|
||||||
### Tokens IN <> Tokens OUT
|
### Tokens IN <> Tokens OUT APIs
|
||||||
|
|
||||||
- `/inference/v1/generate` - Generate completions
|
- `/inference/v1/generate` - Generate completions
|
||||||
- `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)
|
- `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)
|
||||||
|
|||||||
@@ -17,10 +17,9 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
|||||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.renderers.hf import HfRenderer
|
from vllm.renderers.hf import HfRenderer
|
||||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||||
from vllm.v1.engine.async_llm import AsyncLLM
|
from vllm.v1.engine.async_llm import AsyncLLM
|
||||||
@@ -206,15 +205,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
|
|||||||
chat_template=None,
|
chat_template=None,
|
||||||
chat_template_content_format="auto",
|
chat_template_content_format="auto",
|
||||||
)
|
)
|
||||||
online_derenderer = OnlineDerenderer(
|
|
||||||
model_config=engine.model_config,
|
|
||||||
renderer=engine.renderer,
|
|
||||||
request_logger=None,
|
|
||||||
chat_template=None,
|
|
||||||
chat_template_content_format="auto",
|
|
||||||
)
|
|
||||||
|
|
||||||
serving_render = ServingRender(models, online_renderer, online_derenderer)
|
serving_render = ServingRender(models, online_renderer)
|
||||||
|
|
||||||
async def _fake_preprocess_chat(*args, **kwargs):
|
async def _fake_preprocess_chat(*args, **kwargs):
|
||||||
# return conversation, engine_inputs
|
# return conversation, engine_inputs
|
||||||
|
|||||||
@@ -14,10 +14,9 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
|||||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.renderers.hf import HfRenderer
|
from vllm.renderers.hf import HfRenderer
|
||||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||||
from vllm.v1.engine.async_llm import AsyncLLM
|
from vllm.v1.engine.async_llm import AsyncLLM
|
||||||
@@ -191,15 +190,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
|
|||||||
chat_template=None,
|
chat_template=None,
|
||||||
chat_template_content_format="auto",
|
chat_template_content_format="auto",
|
||||||
)
|
)
|
||||||
online_derenderer = OnlineDerenderer(
|
|
||||||
model_config=engine.model_config,
|
|
||||||
renderer=engine.renderer,
|
|
||||||
request_logger=None,
|
|
||||||
chat_template=None,
|
|
||||||
chat_template_content_format="auto",
|
|
||||||
)
|
|
||||||
|
|
||||||
serving_render = ServingRender(models, online_renderer, online_derenderer)
|
serving_render = ServingRender(models, online_renderer)
|
||||||
|
|
||||||
async def _fake_preprocess_chat(*args, **kwargs):
|
async def _fake_preprocess_chat(*args, **kwargs):
|
||||||
# return conversation, engine_inputs
|
# return conversation, engine_inputs
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
"""Unit tests for the token-offsets request/response protocol wiring:
|
"""Unit tests for the token-offsets request/response protocol wiring:
|
||||||
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
|
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
|
||||||
serialization boundary. End-to-end behavior is covered by
|
serialization boundary. End-to-end behavior is covered by
|
||||||
``tests/entrypoints/serve/render/test_render.py``; plain Pydantic field
|
``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field
|
||||||
storage is not retested here.
|
storage is not retested here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -12,7 +12,7 @@ from unittest.mock import Mock
|
|||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -12,11 +12,11 @@ from vllm.config.multimodal import MultiModalConfig
|
|||||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
GenerateRequest,
|
GenerateRequest,
|
||||||
GenerateResponse,
|
GenerateResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.renderers import renderer_from_config
|
from vllm.renderers import renderer_from_config
|
||||||
+6
-3
@@ -1,14 +1,17 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Roundtrip tests for multimodal serde used by the disagg generate endpoint."""
|
"""
|
||||||
|
Roundtrip tests for multimodal serde used by the
|
||||||
|
token_in_token_out generate endpoint.
|
||||||
|
"""
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.entrypoints.serve.disagg.mm_serde import (
|
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||||
decode_mm_kwargs_item,
|
decode_mm_kwargs_item,
|
||||||
encode_mm_kwargs_item,
|
encode_mm_kwargs_item,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
MultiModalFeatures,
|
MultiModalFeatures,
|
||||||
PlaceholderRangeInfo,
|
PlaceholderRangeInfo,
|
||||||
)
|
)
|
||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Unit tests for the disagg request/response protocol.
|
"""Unit tests for the token_in_token_out request/response protocol.
|
||||||
|
|
||||||
These tests intentionally avoid spinning up a server — they exercise the
|
These tests intentionally avoid spinning up a server — they exercise the
|
||||||
pydantic validators on ``GenerateRequest`` directly so they run fast and
|
pydantic validators on ``GenerateRequest`` directly so they run fast and
|
||||||
@@ -9,7 +9,7 @@ fail loudly if the validator semantics ever drift.
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
|
||||||
|
|
||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
|
|
||||||
|
|
||||||
@@ -5,7 +5,7 @@
|
|||||||
These exercise ``_tokenize_prompt`` (offset extraction + capability/MM
|
These exercise ``_tokenize_prompt`` (offset extraction + capability/MM
|
||||||
gating) and the ``_tokenize_prompt -> _process_tokens -> TokensInput``
|
gating) and the ``_tokenize_prompt -> _process_tokens -> TokensInput``
|
||||||
forwarding chain. Endpoint-level coverage lives in
|
forwarding chain. Endpoint-level coverage lives in
|
||||||
``tests/entrypoints/serve/render/test_render.py``.
|
``tests/entrypoints/scale_out/render/test_render.py``.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|||||||
@@ -66,7 +66,6 @@ async def init_generate_state(
|
|||||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
|
||||||
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
|
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
|
||||||
|
|
||||||
# Applied before any serving class is constructed so that each one picks
|
# Applied before any serving class is constructed so that each one picks
|
||||||
@@ -175,20 +174,6 @@ async def init_generate_state(
|
|||||||
if "generate" in supported_tasks
|
if "generate" in supported_tasks
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
state.serving_tokens = (
|
|
||||||
ServingTokens(
|
|
||||||
engine_client,
|
|
||||||
state.openai_serving_models,
|
|
||||||
state.online_renderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
|
||||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
|
||||||
enable_log_outputs=args.enable_log_outputs,
|
|
||||||
force_no_detokenize=args.tokens_only,
|
|
||||||
)
|
|
||||||
if "generate" in supported_tasks
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
|
|
||||||
from .generative_scoring.serving import ServingGenerativeScoring
|
from .generative_scoring.serving import ServingGenerativeScoring
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,6 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
|
|||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
||||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
|
||||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
@@ -208,12 +207,6 @@ def build_app(
|
|||||||
|
|
||||||
register_generate_api_routers(app)
|
register_generate_api_routers(app)
|
||||||
|
|
||||||
from vllm.entrypoints.serve.disagg.api_router import (
|
|
||||||
attach_router as attach_disagg_router,
|
|
||||||
)
|
|
||||||
|
|
||||||
attach_disagg_router(app)
|
|
||||||
|
|
||||||
from vllm.entrypoints.serve.elastic_ep.api_router import (
|
from vllm.entrypoints.serve.elastic_ep.api_router import (
|
||||||
attach_router as elastic_ep_attach_router,
|
attach_router as elastic_ep_attach_router,
|
||||||
)
|
)
|
||||||
@@ -221,11 +214,9 @@ def build_app(
|
|||||||
elastic_ep_attach_router(app)
|
elastic_ep_attach_router(app)
|
||||||
|
|
||||||
if "generate" in supported_tasks or "render" in supported_tasks:
|
if "generate" in supported_tasks or "render" in supported_tasks:
|
||||||
from vllm.entrypoints.serve.render.api_router import (
|
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||||
attach_router as attach_render_router,
|
|
||||||
)
|
|
||||||
|
|
||||||
attach_render_router(app)
|
register_scale_out_api_routers(app, supported_tasks)
|
||||||
|
|
||||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||||
from vllm.entrypoints.speech_to_text.factories import (
|
from vllm.entrypoints.speech_to_text.factories import (
|
||||||
@@ -401,12 +392,6 @@ async def init_app_state(
|
|||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
)
|
)
|
||||||
state.serving_render = ServingRender(
|
|
||||||
state.openai_serving_models,
|
|
||||||
state.online_renderer,
|
|
||||||
state.online_derenderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
)
|
|
||||||
|
|
||||||
if "generate" in supported_tasks:
|
if "generate" in supported_tasks:
|
||||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||||
@@ -415,6 +400,10 @@ async def init_app_state(
|
|||||||
engine_client, state, args, request_logger, supported_tasks
|
engine_client, state, args, request_logger, supported_tasks
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||||
|
|
||||||
|
init_scale_out_state(state, args, engine_client, request_logger)
|
||||||
|
|
||||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||||
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
||||||
|
|
||||||
@@ -505,12 +494,10 @@ async def init_render_app_state(
|
|||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
)
|
)
|
||||||
state.serving_render = ServingRender(
|
|
||||||
model_registry,
|
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||||
state.online_renderer,
|
|
||||||
state.online_derenderer,
|
init_render_state(state, request_logger)
|
||||||
request_logger=request_logger,
|
|
||||||
)
|
|
||||||
|
|
||||||
state.vllm_config = vllm_config
|
state.vllm_config = vllm_config
|
||||||
# Disable stats logging — there is no engine to poll.
|
# Disable stats logging — there is no engine to poll.
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from http import HTTPStatus
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Request
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||||
|
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
|
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
from ..token_in_token_out.protocol import (
|
||||||
|
DerenderChatRequest,
|
||||||
|
DerenderCompletionRequest,
|
||||||
|
)
|
||||||
|
from .serving import ServingDerender
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
def derender(request: Request) -> ServingDerender | None:
|
||||||
|
return getattr(request.app.state, "serving_derender", None)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/v1/chat/completions/derender",
|
||||||
|
dependencies=[Depends(validate_json_request)],
|
||||||
|
response_model=ChatCompletionResponse,
|
||||||
|
responses={
|
||||||
|
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
|
||||||
|
handler = derender(raw_request)
|
||||||
|
if handler is None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
"The model does not support Chat Completions Derender API"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await handler.derender_chat_response(request)
|
||||||
|
|
||||||
|
if isinstance(result, ErrorResponse):
|
||||||
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
|
|
||||||
|
return JSONResponse(content=result.model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/v1/completions/derender",
|
||||||
|
dependencies=[Depends(validate_json_request)],
|
||||||
|
response_model=CompletionResponse,
|
||||||
|
responses={
|
||||||
|
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
|
||||||
|
handler = derender(raw_request)
|
||||||
|
if handler is None:
|
||||||
|
raise NotImplementedError("The model does not support Completions Derender API")
|
||||||
|
|
||||||
|
result = await handler.derender_completion_response(request)
|
||||||
|
|
||||||
|
if isinstance(result, ErrorResponse):
|
||||||
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
|
|
||||||
|
return JSONResponse(content=result.model_dump())
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import time
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||||
|
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import (
|
||||||
|
ErrorResponse,
|
||||||
|
UsageInfo,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.openai.models.serving import (
|
||||||
|
OpenAIModelRegistry,
|
||||||
|
OpenAIServingModels,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||||
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
|
from vllm.inputs import (
|
||||||
|
EngineInput,
|
||||||
|
MultiModalHashes,
|
||||||
|
MultiModalInput,
|
||||||
|
MultiModalPlaceholders,
|
||||||
|
)
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||||
|
|
||||||
|
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
|
||||||
|
from ..token_in_token_out.protocol import (
|
||||||
|
DerenderChatRequest,
|
||||||
|
DerenderCompletionRequest,
|
||||||
|
MultiModalFeatures,
|
||||||
|
PlaceholderRangeInfo,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ServingDerender(BaseServing):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||||
|
online_derenderer: "OnlineDerenderer",
|
||||||
|
*,
|
||||||
|
request_logger: RequestLogger | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(
|
||||||
|
models=models,
|
||||||
|
model_config=models.model_config,
|
||||||
|
request_logger=request_logger,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.online_derenderer = online_derenderer
|
||||||
|
|
||||||
|
async def derender_chat_response(
|
||||||
|
self,
|
||||||
|
request: DerenderChatRequest,
|
||||||
|
) -> ChatCompletionResponse | ErrorResponse:
|
||||||
|
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||||
|
|
||||||
|
Non-streaming only: expects the complete GenerateResponse with all
|
||||||
|
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
|
||||||
|
|
||||||
|
When ``request.chat_request`` is provided, the parser splits the
|
||||||
|
output into (reasoning, content, tool_calls). Otherwise falls
|
||||||
|
back to plain detokenization.
|
||||||
|
"""
|
||||||
|
error_check_ret = await self._check_model(request)
|
||||||
|
if error_check_ret is not None:
|
||||||
|
return error_check_ret
|
||||||
|
|
||||||
|
try:
|
||||||
|
choices = await self.online_derenderer.derender_chat(
|
||||||
|
request.generate_response, request.chat_request
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
return self.create_error_response(str(exc))
|
||||||
|
|
||||||
|
prompt_tokens = (
|
||||||
|
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||||
|
)
|
||||||
|
gen = request.generate_response
|
||||||
|
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||||
|
usage = UsageInfo(
|
||||||
|
prompt_tokens=prompt_tokens,
|
||||||
|
completion_tokens=completion_tokens,
|
||||||
|
total_tokens=prompt_tokens + completion_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||||
|
gen.request_id,
|
||||||
|
request.model,
|
||||||
|
len(choices),
|
||||||
|
completion_tokens,
|
||||||
|
)
|
||||||
|
return ChatCompletionResponse(
|
||||||
|
id=gen.request_id,
|
||||||
|
model=request.model,
|
||||||
|
created=int(time.time()),
|
||||||
|
choices=choices,
|
||||||
|
usage=usage,
|
||||||
|
prompt_logprobs=gen.prompt_logprobs,
|
||||||
|
kv_transfer_params=gen.kv_transfer_params,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def derender_completion_response(
|
||||||
|
self,
|
||||||
|
request: DerenderCompletionRequest,
|
||||||
|
) -> CompletionResponse | ErrorResponse:
|
||||||
|
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||||
|
|
||||||
|
Non-streaming only. Mirrors the multi-prompt completions case: one
|
||||||
|
GenerateResponse per prompt, parallel to the list[GenerateRequest]
|
||||||
|
from /v1/completions/render.
|
||||||
|
"""
|
||||||
|
error_check_ret = await self._check_model(request)
|
||||||
|
if error_check_ret is not None:
|
||||||
|
return error_check_ret
|
||||||
|
|
||||||
|
(
|
||||||
|
choices,
|
||||||
|
total_prompt_tokens,
|
||||||
|
total_completion_tokens,
|
||||||
|
) = await self.online_derenderer.derender_completion(
|
||||||
|
request.generate_responses, request.prompt_tokens
|
||||||
|
)
|
||||||
|
|
||||||
|
if not request.generate_responses:
|
||||||
|
return self.create_error_response("generate_responses must not be empty")
|
||||||
|
|
||||||
|
first = request.generate_responses[0]
|
||||||
|
kv_params = first.kv_transfer_params
|
||||||
|
if any(
|
||||||
|
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||||
|
):
|
||||||
|
logger.warning(
|
||||||
|
"derender_completion: kv_transfer_params differ across responses; "
|
||||||
|
"setting to None on the aggregated response"
|
||||||
|
)
|
||||||
|
kv_params = None
|
||||||
|
|
||||||
|
usage = UsageInfo(
|
||||||
|
prompt_tokens=total_prompt_tokens,
|
||||||
|
completion_tokens=total_completion_tokens,
|
||||||
|
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"derender_completion request_id=%s model=%s choices=%d"
|
||||||
|
" completion_tokens=%d",
|
||||||
|
first.request_id,
|
||||||
|
request.model,
|
||||||
|
len(choices),
|
||||||
|
total_completion_tokens,
|
||||||
|
)
|
||||||
|
return CompletionResponse(
|
||||||
|
id=first.request_id,
|
||||||
|
model=request.model,
|
||||||
|
created=int(time.time()),
|
||||||
|
choices=choices,
|
||||||
|
usage=usage,
|
||||||
|
kv_transfer_params=kv_params,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_mm_features(
|
||||||
|
engine_input: EngineInput,
|
||||||
|
) -> MultiModalFeatures | None:
|
||||||
|
"""Extract multimodal metadata from a rendered engine prompt.
|
||||||
|
|
||||||
|
Returns ``None`` for text-only prompts.
|
||||||
|
"""
|
||||||
|
if engine_input.get("type") != "multimodal":
|
||||||
|
return None
|
||||||
|
|
||||||
|
# At this point engine_input is a MultiModalInput TypedDict.
|
||||||
|
mm_engine_input = cast(MultiModalInput, engine_input)
|
||||||
|
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
|
||||||
|
raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
|
||||||
|
|
||||||
|
mm_placeholders = {
|
||||||
|
modality: [
|
||||||
|
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
|
||||||
|
]
|
||||||
|
for modality, ranges in raw_placeholders.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
# Serialize tensor data per modality.
|
||||||
|
kwargs_data: dict[str, list[str | None]] | None = None
|
||||||
|
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
|
||||||
|
kwargs_data = {}
|
||||||
|
for modality, items in raw_mm_kwargs.items():
|
||||||
|
kwargs_data[modality] = [
|
||||||
|
encode_mm_kwargs_item(item) if item is not None else None
|
||||||
|
for item in items
|
||||||
|
]
|
||||||
|
|
||||||
|
return MultiModalFeatures(
|
||||||
|
mm_hashes=mm_hashes,
|
||||||
|
mm_placeholders=mm_placeholders,
|
||||||
|
kwargs_data=kwargs_data,
|
||||||
|
)
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from argparse import Namespace
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
|
||||||
|
from vllm.engine.protocol import EngineClient
|
||||||
|
from vllm.tasks import SupportedTask
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from starlette.datastructures import State
|
||||||
|
|
||||||
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
|
else:
|
||||||
|
RequestLogger = object
|
||||||
|
|
||||||
|
|
||||||
|
def init_render_state(
|
||||||
|
state: "State",
|
||||||
|
request_logger: RequestLogger | None,
|
||||||
|
):
|
||||||
|
from .derender.serving import ServingDerender
|
||||||
|
from .render.serving import ServingRender
|
||||||
|
|
||||||
|
state.serving_render = ServingRender(
|
||||||
|
state.openai_serving_models,
|
||||||
|
state.online_renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
)
|
||||||
|
|
||||||
|
state.serving_derender = ServingDerender(
|
||||||
|
state.openai_serving_models,
|
||||||
|
state.online_derenderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def init_scale_out_state(
|
||||||
|
state: "State",
|
||||||
|
args: "Namespace",
|
||||||
|
engine_client: "EngineClient",
|
||||||
|
request_logger: RequestLogger | None,
|
||||||
|
):
|
||||||
|
init_render_state(state, request_logger)
|
||||||
|
|
||||||
|
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||||
|
|
||||||
|
state.serving_tokens = ServingTokens(
|
||||||
|
engine_client,
|
||||||
|
state.openai_serving_models,
|
||||||
|
state.online_renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||||
|
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||||
|
enable_log_outputs=args.enable_log_outputs,
|
||||||
|
force_no_detokenize=args.tokens_only,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register_scale_out_api_routers(
|
||||||
|
app: FastAPI,
|
||||||
|
supported_tasks: tuple["SupportedTask", ...],
|
||||||
|
):
|
||||||
|
from .render.api_router import router as render_render
|
||||||
|
|
||||||
|
app.include_router(render_render)
|
||||||
|
|
||||||
|
from .derender.api_router import router as derender_render
|
||||||
|
|
||||||
|
app.include_router(derender_render)
|
||||||
|
|
||||||
|
if "generate" in supported_tasks:
|
||||||
|
from .token_in_token_out.api_router import (
|
||||||
|
attach_router as attach_disagg_router,
|
||||||
|
)
|
||||||
|
|
||||||
|
attach_disagg_router(app)
|
||||||
+6
-67
@@ -2,27 +2,18 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
from fastapi import APIRouter, Depends, Request
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
ChatCompletionRequest,
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
ChatCompletionResponse,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.completion.protocol import (
|
|
||||||
CompletionRequest,
|
|
||||||
CompletionResponse,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
|
||||||
DerenderChatRequest,
|
|
||||||
DerenderCompletionRequest,
|
|
||||||
GenerateRequest,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
from ..token_in_token_out.protocol import GenerateRequest
|
||||||
|
from .serving import ServingRender
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
@@ -79,55 +70,3 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
|
|||||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
|
|
||||||
return JSONResponse(content=[item.model_dump() for item in result])
|
return JSONResponse(content=[item.model_dump() for item in result])
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
|
||||||
"/v1/chat/completions/derender",
|
|
||||||
dependencies=[Depends(validate_json_request)],
|
|
||||||
response_model=ChatCompletionResponse,
|
|
||||||
responses={
|
|
||||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
|
||||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
|
||||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
|
|
||||||
handler = render(raw_request)
|
|
||||||
if handler is None:
|
|
||||||
raise NotImplementedError(
|
|
||||||
"The model does not support Chat Completions Derender API"
|
|
||||||
)
|
|
||||||
|
|
||||||
result = await handler.derender_chat_response(request)
|
|
||||||
|
|
||||||
if isinstance(result, ErrorResponse):
|
|
||||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
|
||||||
|
|
||||||
return JSONResponse(content=result.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
|
||||||
"/v1/completions/derender",
|
|
||||||
dependencies=[Depends(validate_json_request)],
|
|
||||||
response_model=CompletionResponse,
|
|
||||||
responses={
|
|
||||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
|
||||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
|
||||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
|
|
||||||
handler = render(raw_request)
|
|
||||||
if handler is None:
|
|
||||||
raise NotImplementedError("The model does not support Completions Derender API")
|
|
||||||
|
|
||||||
result = await handler.derender_completion_response(request)
|
|
||||||
|
|
||||||
if isinstance(result, ErrorResponse):
|
|
||||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
|
||||||
|
|
||||||
return JSONResponse(content=result.model_dump())
|
|
||||||
|
|
||||||
|
|
||||||
def attach_router(app: FastAPI) -> None:
|
|
||||||
app.include_router(router)
|
|
||||||
+5
-131
@@ -1,28 +1,16 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import time
|
|
||||||
from typing import cast
|
from typing import cast
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
ChatCompletionRequest,
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
ChatCompletionResponse,
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.completion.protocol import (
|
|
||||||
CompletionRequest,
|
|
||||||
CompletionResponse,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import (
|
|
||||||
ErrorResponse,
|
|
||||||
UsageInfo,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.models.serving import (
|
from vllm.entrypoints.openai.models.serving import (
|
||||||
OpenAIModelRegistry,
|
OpenAIModelRegistry,
|
||||||
OpenAIServingModels,
|
OpenAIServingModels,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
|
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import encode_mm_kwargs_item
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
DerenderChatRequest,
|
|
||||||
DerenderCompletionRequest,
|
|
||||||
GenerateRequest,
|
GenerateRequest,
|
||||||
MultiModalFeatures,
|
MultiModalFeatures,
|
||||||
PlaceholderRangeInfo,
|
PlaceholderRangeInfo,
|
||||||
@@ -41,7 +29,6 @@ from vllm.renderers.inputs.preprocess import (
|
|||||||
extract_prompt_components,
|
extract_prompt_components,
|
||||||
extract_prompt_len,
|
extract_prompt_len,
|
||||||
)
|
)
|
||||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
|
||||||
@@ -53,7 +40,6 @@ class ServingRender(BaseServing):
|
|||||||
self,
|
self,
|
||||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||||
online_renderer: "OnlineRenderer",
|
online_renderer: "OnlineRenderer",
|
||||||
online_derenderer: "OnlineDerenderer",
|
|
||||||
*,
|
*,
|
||||||
request_logger: RequestLogger | None = None,
|
request_logger: RequestLogger | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -64,7 +50,6 @@ class ServingRender(BaseServing):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.online_renderer = online_renderer
|
self.online_renderer = online_renderer
|
||||||
self.online_derenderer = online_derenderer
|
|
||||||
|
|
||||||
self.default_sampling_params = (
|
self.default_sampling_params = (
|
||||||
online_renderer.model_config.get_diff_sampling_param()
|
online_renderer.model_config.get_diff_sampling_param()
|
||||||
@@ -223,117 +208,6 @@ class ServingRender(BaseServing):
|
|||||||
|
|
||||||
return generate_requests
|
return generate_requests
|
||||||
|
|
||||||
async def derender_chat_response(
|
|
||||||
self,
|
|
||||||
request: DerenderChatRequest,
|
|
||||||
) -> ChatCompletionResponse | ErrorResponse:
|
|
||||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
|
||||||
|
|
||||||
Non-streaming only: expects the complete GenerateResponse with all
|
|
||||||
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
|
|
||||||
|
|
||||||
When ``request.chat_request`` is provided, the parser splits the
|
|
||||||
output into (reasoning, content, tool_calls). Otherwise falls
|
|
||||||
back to plain detokenization.
|
|
||||||
"""
|
|
||||||
error_check_ret = await self._check_model(request)
|
|
||||||
if error_check_ret is not None:
|
|
||||||
return error_check_ret
|
|
||||||
|
|
||||||
try:
|
|
||||||
choices = await self.online_derenderer.derender_chat(
|
|
||||||
request.generate_response, request.chat_request
|
|
||||||
)
|
|
||||||
except ValueError as exc:
|
|
||||||
return self.create_error_response(str(exc))
|
|
||||||
|
|
||||||
prompt_tokens = (
|
|
||||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
|
||||||
)
|
|
||||||
gen = request.generate_response
|
|
||||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
|
||||||
usage = UsageInfo(
|
|
||||||
prompt_tokens=prompt_tokens,
|
|
||||||
completion_tokens=completion_tokens,
|
|
||||||
total_tokens=prompt_tokens + completion_tokens,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
|
||||||
gen.request_id,
|
|
||||||
request.model,
|
|
||||||
len(choices),
|
|
||||||
completion_tokens,
|
|
||||||
)
|
|
||||||
return ChatCompletionResponse(
|
|
||||||
id=gen.request_id,
|
|
||||||
model=request.model,
|
|
||||||
created=int(time.time()),
|
|
||||||
choices=choices,
|
|
||||||
usage=usage,
|
|
||||||
prompt_logprobs=gen.prompt_logprobs,
|
|
||||||
kv_transfer_params=gen.kv_transfer_params,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def derender_completion_response(
|
|
||||||
self,
|
|
||||||
request: DerenderCompletionRequest,
|
|
||||||
) -> CompletionResponse | ErrorResponse:
|
|
||||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
|
||||||
|
|
||||||
Non-streaming only. Mirrors the multi-prompt completions case: one
|
|
||||||
GenerateResponse per prompt, parallel to the list[GenerateRequest]
|
|
||||||
from /v1/completions/render.
|
|
||||||
"""
|
|
||||||
error_check_ret = await self._check_model(request)
|
|
||||||
if error_check_ret is not None:
|
|
||||||
return error_check_ret
|
|
||||||
|
|
||||||
(
|
|
||||||
choices,
|
|
||||||
total_prompt_tokens,
|
|
||||||
total_completion_tokens,
|
|
||||||
) = await self.online_derenderer.derender_completion(
|
|
||||||
request.generate_responses, request.prompt_tokens
|
|
||||||
)
|
|
||||||
|
|
||||||
if not request.generate_responses:
|
|
||||||
return self.create_error_response("generate_responses must not be empty")
|
|
||||||
|
|
||||||
first = request.generate_responses[0]
|
|
||||||
kv_params = first.kv_transfer_params
|
|
||||||
if any(
|
|
||||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
|
||||||
):
|
|
||||||
logger.warning(
|
|
||||||
"derender_completion: kv_transfer_params differ across responses; "
|
|
||||||
"setting to None on the aggregated response"
|
|
||||||
)
|
|
||||||
kv_params = None
|
|
||||||
|
|
||||||
usage = UsageInfo(
|
|
||||||
prompt_tokens=total_prompt_tokens,
|
|
||||||
completion_tokens=total_completion_tokens,
|
|
||||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
"derender_completion request_id=%s model=%s choices=%d"
|
|
||||||
" completion_tokens=%d",
|
|
||||||
first.request_id,
|
|
||||||
request.model,
|
|
||||||
len(choices),
|
|
||||||
total_completion_tokens,
|
|
||||||
)
|
|
||||||
return CompletionResponse(
|
|
||||||
id=first.request_id,
|
|
||||||
model=request.model,
|
|
||||||
created=int(time.time()),
|
|
||||||
choices=choices,
|
|
||||||
usage=usage,
|
|
||||||
kv_transfer_params=kv_params,
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_mm_features(
|
def _extract_mm_features(
|
||||||
engine_input: EngineInput,
|
engine_input: EngineInput,
|
||||||
+6
-7
@@ -11,13 +11,6 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
|
||||||
GenerateRequest,
|
|
||||||
GenerateResponse,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.serve.disagg.serving import (
|
|
||||||
ServingTokens,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
load_aware_call,
|
load_aware_call,
|
||||||
@@ -26,6 +19,12 @@ from vllm.entrypoints.serve.utils.api_utils import (
|
|||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
from .protocol import (
|
||||||
|
GenerateRequest,
|
||||||
|
GenerateResponse,
|
||||||
|
)
|
||||||
|
from .serving import ServingTokens
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
+9
-8
@@ -28,14 +28,6 @@ from vllm.entrypoints.openai.engine.protocol import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item
|
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
|
||||||
GenerateRequest,
|
|
||||||
GenerateResponse,
|
|
||||||
GenerateResponseChoice,
|
|
||||||
GenerateResponseStreamChoice,
|
|
||||||
GenerateStreamResponse,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.inputs import EngineInput, mm_input
|
from vllm.inputs import EngineInput, mm_input
|
||||||
@@ -51,6 +43,15 @@ from vllm.renderers.online_renderer import OnlineRenderer
|
|||||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||||
from vllm.utils.collection_utils import as_list
|
from vllm.utils.collection_utils import as_list
|
||||||
|
|
||||||
|
from .mm_serde import decode_mm_kwargs_item
|
||||||
|
from .protocol import (
|
||||||
|
GenerateRequest,
|
||||||
|
GenerateResponse,
|
||||||
|
GenerateResponseChoice,
|
||||||
|
GenerateResponseStreamChoice,
|
||||||
|
GenerateStreamResponse,
|
||||||
|
)
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -15,7 +15,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
|||||||
CompletionResponse,
|
CompletionResponse,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.entrypoints.serve.disagg.protocol import (
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||||
DerenderChatRequest,
|
DerenderChatRequest,
|
||||||
DerenderCompletionRequest,
|
DerenderCompletionRequest,
|
||||||
GenerateRequest,
|
GenerateRequest,
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.engine.protocol import ToolCall
|
from vllm.entrypoints.openai.engine.protocol import ToolCall
|
||||||
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
|
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
|
||||||
from vllm.entrypoints.serve.disagg.protocol import GenerateResponse
|
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.parser import Parser, ParserManager
|
from vllm.parser import Parser, ParserManager
|
||||||
|
|||||||
Reference in New Issue
Block a user