[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -8,7 +8,6 @@ import time
|
||||
import uvloop
|
||||
|
||||
import vllm
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
@@ -284,11 +283,6 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
usage_context = UsageContext.OPENAI_API_SERVER
|
||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||
|
||||
if num_api_servers > 1 and envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
|
||||
raise ValueError(
|
||||
"VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used with api_server_count > 1"
|
||||
)
|
||||
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
log_stats = not engine_args.disable_log_stats
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
@@ -55,58 +54,6 @@ class OfflineInferenceMixin:
|
||||
model_config: ModelConfig
|
||||
chat_template: str | None
|
||||
|
||||
def _resolve_mm_lora(
|
||||
self,
|
||||
prompt: EngineInput,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> LoRARequest | None:
|
||||
if prompt["type"] != "multimodal":
|
||||
return lora_request
|
||||
|
||||
lora_config = self.llm_engine.vllm_config.lora_config
|
||||
default_mm_loras = None if lora_config is None else lora_config.default_mm_loras
|
||||
if not default_mm_loras:
|
||||
return lora_request
|
||||
|
||||
prompt_modalities = prompt["mm_placeholders"].keys()
|
||||
intersection = set(prompt_modalities).intersection(default_mm_loras.keys())
|
||||
if not intersection:
|
||||
return lora_request
|
||||
|
||||
if len(intersection) > 1:
|
||||
# TODO: Would be nice to be able to have multiple loras per prompt
|
||||
logger.warning(
|
||||
"Multiple modality specific loras were registered and would be "
|
||||
"used by a single prompt consuming several modalities; "
|
||||
"currently we only support one lora per request; as such, "
|
||||
"lora(s) registered with modalities: %s will be skipped",
|
||||
intersection,
|
||||
)
|
||||
return lora_request
|
||||
|
||||
# Build the LoRA request; the ID of the default mm lora is the
|
||||
# index of the modality name sorted alphabetically + 1.
|
||||
modality_name = intersection.pop()
|
||||
modality_lora_path = default_mm_loras[modality_name]
|
||||
modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
|
||||
|
||||
# If we have a collision, warn if there is a collision,
|
||||
# but always send the explicitly provided request.
|
||||
if lora_request:
|
||||
if lora_request.lora_int_id != modality_lora_id:
|
||||
logger.warning(
|
||||
"A modality with a registered lora and a lora_request "
|
||||
"with a different ID were provided; falling back to the "
|
||||
"lora_request as we only apply one LoRARequest per prompt"
|
||||
)
|
||||
return lora_request
|
||||
|
||||
return LoRARequest(
|
||||
modality_name,
|
||||
modality_lora_id,
|
||||
modality_lora_path,
|
||||
)
|
||||
|
||||
def _preprocess_cmpl(
|
||||
self,
|
||||
prompts: Sequence[PromptType],
|
||||
@@ -262,22 +209,6 @@ class OfflineInferenceMixin:
|
||||
|
||||
return [params] * num_requests
|
||||
|
||||
def _lora_request_to_seq(
|
||||
self,
|
||||
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
|
||||
num_requests: int,
|
||||
) -> Sequence[LoRARequest | None]:
|
||||
if isinstance(lora_request, Sequence):
|
||||
if len(lora_request) != num_requests:
|
||||
raise VLLMValidationError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and lora_request ({len(lora_request)}) must be the same."
|
||||
)
|
||||
|
||||
return lora_request
|
||||
|
||||
return [lora_request] * num_requests
|
||||
|
||||
def _priority_to_seq(
|
||||
self,
|
||||
priority: int | Sequence[int] | None,
|
||||
@@ -305,14 +236,12 @@ class OfflineInferenceMixin:
|
||||
| Sequence[SamplingParams | PoolingParams],
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[str]:
|
||||
seq_prompts = prompt_to_seq(prompts)
|
||||
seq_params = self._params_to_seq(params, len(seq_prompts))
|
||||
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))
|
||||
seq_priority = self._priority_to_seq(priority, len(seq_prompts))
|
||||
|
||||
return self._render_and_add_requests(
|
||||
@@ -329,7 +258,6 @@ class OfflineInferenceMixin:
|
||||
)
|
||||
),
|
||||
params=seq_params,
|
||||
lora_requests=seq_lora_requests,
|
||||
priorities=seq_priority,
|
||||
)
|
||||
|
||||
@@ -342,7 +270,6 @@ class OfflineInferenceMixin:
|
||||
output_type: type[_O],
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
@@ -351,7 +278,6 @@ class OfflineInferenceMixin:
|
||||
prompts=prompts,
|
||||
params=params,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
@@ -368,7 +294,6 @@ class OfflineInferenceMixin:
|
||||
output_type: type[_O],
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
@@ -383,7 +308,6 @@ class OfflineInferenceMixin:
|
||||
messages=messages,
|
||||
params=params,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
@@ -405,7 +329,6 @@ class OfflineInferenceMixin:
|
||||
| Sequence[SamplingParams | PoolingParams],
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
@@ -418,7 +341,6 @@ class OfflineInferenceMixin:
|
||||
) -> list[str]:
|
||||
seq_convs = conversation_to_seq(messages)
|
||||
seq_params = self._params_to_seq(params, len(seq_convs))
|
||||
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
|
||||
seq_priority = self._priority_to_seq(priority, len(seq_convs))
|
||||
|
||||
# When thinking is enabled or tools are provided, and the model
|
||||
@@ -452,7 +374,6 @@ class OfflineInferenceMixin:
|
||||
)
|
||||
),
|
||||
params=seq_params,
|
||||
lora_requests=seq_lora_requests,
|
||||
priorities=seq_priority,
|
||||
)
|
||||
|
||||
@@ -509,7 +430,6 @@ class OfflineInferenceMixin:
|
||||
params: Sequence[SamplingParams | PoolingParams],
|
||||
output_type: type[_O],
|
||||
*,
|
||||
lora_requests: Sequence[LoRARequest | None] | None = None,
|
||||
priorities: Sequence[int] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
):
|
||||
@@ -526,7 +446,6 @@ class OfflineInferenceMixin:
|
||||
self._render_and_add_requests(
|
||||
prompts=prompts,
|
||||
params=params,
|
||||
lora_requests=lora_requests,
|
||||
priorities=priorities,
|
||||
)
|
||||
|
||||
@@ -537,7 +456,6 @@ class OfflineInferenceMixin:
|
||||
prompts: Iterable[EngineInput],
|
||||
params: Sequence[SamplingParams | PoolingParams],
|
||||
*,
|
||||
lora_requests: Sequence[LoRARequest | None] | None = None,
|
||||
priorities: Sequence[int] | None = None,
|
||||
) -> list[str]:
|
||||
added_request_ids: list[str] = []
|
||||
@@ -547,10 +465,6 @@ class OfflineInferenceMixin:
|
||||
request_id = self._add_request(
|
||||
prompt,
|
||||
params[i],
|
||||
lora_request=self._resolve_mm_lora(
|
||||
prompt,
|
||||
None if lora_requests is None else lora_requests[i],
|
||||
),
|
||||
priority=0 if priorities is None else priorities[i],
|
||||
)
|
||||
added_request_ids.append(request_id)
|
||||
@@ -565,7 +479,6 @@ class OfflineInferenceMixin:
|
||||
self,
|
||||
prompt: EngineInput,
|
||||
params: SamplingParams | PoolingParams,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
) -> str:
|
||||
if isinstance(params, SamplingParams):
|
||||
@@ -578,7 +491,6 @@ class OfflineInferenceMixin:
|
||||
request_id,
|
||||
prompt,
|
||||
params,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
@@ -121,7 +120,6 @@ class ServeContext(Generic[RequestT]):
|
||||
model_name: str
|
||||
request_id: str
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
lora_request: LoRARequest | None = None
|
||||
engine_inputs: list[EngineInput] | None = None
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from tqdm import tqdm
|
||||
from vllm import RequestOutput, TextPrompt, TokensPrompt
|
||||
from vllm.entrypoints.common.offline import OfflineInferenceMixin
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import (
|
||||
BeamSearchParams,
|
||||
@@ -60,7 +59,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
self,
|
||||
prompts: list[TokensPrompt | TextPrompt],
|
||||
params: BeamSearchParams,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
use_tqdm: bool = False,
|
||||
concurrency_limit: int | None = None,
|
||||
) -> list[BeamSearchOutput]:
|
||||
@@ -70,7 +68,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
prompts: A list of prompts. Each prompt can be a string or a list
|
||||
of token IDs.
|
||||
params: The beam search parameters.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
use_tqdm: Whether to use tqdm to display the progress bar.
|
||||
concurrency_limit: The maximum number of concurrent requests.
|
||||
If None, the number of concurrent requests is unlimited.
|
||||
@@ -90,7 +87,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
|
||||
|
||||
engine_inputs = self._preprocess_cmpl(prompts)
|
||||
lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
|
||||
|
||||
if use_tqdm and concurrency_limit is not None:
|
||||
logger.warning(
|
||||
@@ -131,7 +127,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
|
||||
instances: list[BeamSearchInstance] = []
|
||||
|
||||
for lora_req, prompt in zip(lora_requests, engine_inputs):
|
||||
for prompt in engine_inputs:
|
||||
if prompt["type"] == "embeds":
|
||||
raise NotImplementedError(
|
||||
"Embedding prompt not supported for beam search"
|
||||
@@ -140,7 +136,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
instances.append(
|
||||
BeamSearchInstance(
|
||||
prompt,
|
||||
lora_request=lora_req,
|
||||
logprobs=None,
|
||||
),
|
||||
)
|
||||
@@ -279,7 +274,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
prompts=(beam.get_prompt() for beam in active_beams),
|
||||
params=active_params,
|
||||
output_type=RequestOutput,
|
||||
lora_requests=[beam.lora_request for beam in active_beams],
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
@@ -346,7 +340,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
current_beam.orig_prompt,
|
||||
tokens=current_beam.tokens + [token_id],
|
||||
logprobs=current_beam.logprobs + [logprobs],
|
||||
lora_request=current_beam.lora_request,
|
||||
cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import numpy as np
|
||||
from vllm import CompletionOutput, RequestOutput
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
@@ -34,7 +33,6 @@ class BeamSearchOnlineMixin(ABC):
|
||||
prompt: EngineInput,
|
||||
request_id: str,
|
||||
params: BeamSearchParams,
|
||||
lora_request: LoRARequest | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
@@ -80,7 +78,6 @@ class BeamSearchOnlineMixin(ABC):
|
||||
tokens=prompt_token_ids,
|
||||
cum_logprob=0,
|
||||
logprobs=[],
|
||||
lora_request=lora_request,
|
||||
)
|
||||
]
|
||||
completed = []
|
||||
@@ -91,7 +88,6 @@ class BeamSearchOnlineMixin(ABC):
|
||||
|
||||
for i, beam in enumerate(all_beams):
|
||||
prompt_item = beam.get_prompt()
|
||||
lora_request_item = beam.lora_request
|
||||
request_id_item = f"{request_id_batch}-beam-{i}"
|
||||
task = asyncio.create_task(
|
||||
collect_from_async_generator(
|
||||
@@ -99,7 +95,6 @@ class BeamSearchOnlineMixin(ABC):
|
||||
prompt_item,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request_item,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
)
|
||||
@@ -205,7 +200,6 @@ class BeamSearchOnlineMixin(ABC):
|
||||
orig_prompt=prompt,
|
||||
tokens=current_beam.tokens + [token_id],
|
||||
logprobs=current_beam.logprobs + [logprobs],
|
||||
lora_request=current_beam.lora_request,
|
||||
cum_logprob=cum_logprob,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -12,7 +12,6 @@ from vllm.inputs import (
|
||||
tokens_input,
|
||||
)
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -28,7 +27,6 @@ class BeamSearchSequence:
|
||||
# NOTE: Tokens represents decoder tokens in the encoder / decoder case
|
||||
tokens: list[int]
|
||||
logprobs: list[dict[int, Logprob]]
|
||||
lora_request: LoRARequest | None = None
|
||||
cum_logprob: float = 0.0
|
||||
text: str | None = None
|
||||
finish_reason: str | None = None
|
||||
@@ -113,7 +111,6 @@ class BeamSearchInstance:
|
||||
def __init__(
|
||||
self,
|
||||
prompt: TokensInput | MultiModalInput | EncoderDecoderInput,
|
||||
lora_request: LoRARequest | None = None,
|
||||
logprobs: list[dict[int, Logprob]] | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
@@ -127,7 +124,6 @@ class BeamSearchInstance:
|
||||
orig_prompt=prompt,
|
||||
tokens=initial_tokens,
|
||||
logprobs=[] if logprobs is None else list(logprobs),
|
||||
lora_request=lora_request,
|
||||
**kwargs,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -245,12 +245,6 @@ class ServingGenerativeScoring(BaseServing):
|
||||
# Note: Mixed item types (string and token list) are validated by
|
||||
# Pydantic at request parsing time, so we don't need to check here.
|
||||
|
||||
try:
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
except (ValueError, TypeError, RuntimeError) as e:
|
||||
logger.exception("Error preparing request components")
|
||||
return self.create_error_response(e)
|
||||
|
||||
base_id = self._base_request_id(raw_request, default=request.request_id)
|
||||
request_id = f"generative-scoring-{base_id}"
|
||||
created_time = int(time.time())
|
||||
@@ -290,7 +284,6 @@ class ServingGenerativeScoring(BaseServing):
|
||||
f"{request_id}-{i}",
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -299,7 +292,6 @@ class ServingGenerativeScoring(BaseServing):
|
||||
engine_inputs,
|
||||
[sampling_params] * len(engine_inputs),
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
)
|
||||
@@ -343,7 +335,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
total_completion_tokens += len(read.result.outputs[0].token_ids)
|
||||
|
||||
# Build response
|
||||
model_name = self.models.model_name(lora_request)
|
||||
model_name = self.models.model_name()
|
||||
response = GenerativeScoringResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
|
||||
@@ -8,7 +8,6 @@ from dataclasses import dataclass
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
@@ -28,7 +27,6 @@ async def next_token_label_reads(
|
||||
sampling_params: Sequence[SamplingParams],
|
||||
request_id: str,
|
||||
*,
|
||||
lora_request: LoRARequest | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
) -> list[LabelRead]:
|
||||
@@ -41,7 +39,6 @@ async def next_token_label_reads(
|
||||
engine_input,
|
||||
params,
|
||||
f"{request_id}-{i}",
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
)
|
||||
|
||||
@@ -11,9 +11,7 @@ from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
@@ -68,20 +66,10 @@ async def init_app_state(
|
||||
await init_tool_server(args) if "generate" in supported_tasks else None
|
||||
)
|
||||
|
||||
# Merge default_mm_loras into the static lora_modules
|
||||
default_mm_loras = (
|
||||
vllm_config.lora_config.default_mm_loras
|
||||
if vllm_config.lora_config is not None
|
||||
else {}
|
||||
)
|
||||
lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
|
||||
|
||||
state.openai_serving_models = OpenAIServingModels(
|
||||
engine_client=engine_client,
|
||||
base_model_paths=base_model_paths,
|
||||
lora_modules=lora_modules,
|
||||
)
|
||||
await state.openai_serving_models.init_static_loras()
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=engine_client.model_config,
|
||||
@@ -123,18 +111,6 @@ async def init_app_state(
|
||||
default_chat_template_kwargs,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||
|
||||
init_scale_out_state(state, args, engine_client, request_logger)
|
||||
|
||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
||||
|
||||
init_speech_to_text_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
await init_endpoint_plugins_state(engine_client, state, args)
|
||||
|
||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||
state.server_load_metrics = 0
|
||||
|
||||
@@ -31,17 +31,6 @@ def register_api_routers(
|
||||
|
||||
register_generate_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||
|
||||
register_scale_out_api_routers(app, supported_tasks)
|
||||
|
||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||
from vllm.entrypoints.speech_to_text.factories import (
|
||||
register_speech_to_text_api_routers,
|
||||
)
|
||||
|
||||
register_speech_to_text_api_routers(app, supported_tasks)
|
||||
|
||||
if getattr(args, "enable_fault_tolerance", False):
|
||||
from vllm.entrypoints.serve.fault_tolerance.api_router import (
|
||||
register_fault_tolerance_api_router,
|
||||
|
||||
@@ -8,7 +8,6 @@ from fastapi import FastAPI
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
||||
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
from .api_server.routers import register_api_routers
|
||||
@@ -41,9 +40,6 @@ def build_app(
|
||||
|
||||
register_api_routers(args, app, supported_tasks, model_config)
|
||||
|
||||
# Endpoint plugin hooks run after core routers so plugins can inspect them.
|
||||
attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
init_exception_handler(app)
|
||||
init_entrypoints_middleware(args, app, supported_tasks)
|
||||
return app
|
||||
|
||||
@@ -7,13 +7,11 @@ It is kept in a separate file for documentation purposes.
|
||||
import argparse
|
||||
import json
|
||||
import ssl
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import field
|
||||
from typing import Any, Literal
|
||||
|
||||
from vllm.config import config
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
@@ -31,40 +29,6 @@ from .utils.constants import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LoRAParserAction(argparse.Action):
|
||||
def __call__(
|
||||
self,
|
||||
parser: argparse.ArgumentParser,
|
||||
namespace: argparse.Namespace,
|
||||
values: str | Sequence[str] | None,
|
||||
option_string: str | None = None,
|
||||
):
|
||||
if values is None:
|
||||
values = []
|
||||
if isinstance(values, str):
|
||||
raise TypeError("Expected values to be a list")
|
||||
|
||||
lora_list: list[LoRAModulePath] = []
|
||||
for item in values:
|
||||
if item in [None, ""]: # Skip if item is None or empty string
|
||||
continue
|
||||
if "=" in item and "," not in item: # Old format: name=path
|
||||
name, path = item.split("=", 1)
|
||||
lora_list.append(LoRAModulePath(name, path))
|
||||
else: # Assume JSON format
|
||||
try:
|
||||
lora_dict = json.loads(item)
|
||||
lora = LoRAModulePath(**lora_dict)
|
||||
lora_list.append(lora)
|
||||
except json.JSONDecodeError:
|
||||
parser.error(f"Invalid JSON format for --lora-modules: {item}")
|
||||
except TypeError as e:
|
||||
parser.error(
|
||||
f"Invalid fields for --lora-modules: {item} - {str(e)}"
|
||||
)
|
||||
setattr(namespace, self.dest, lora_list)
|
||||
|
||||
|
||||
@config
|
||||
class BaseFrontendArgs:
|
||||
"""Base arguments for the OpenAI-compatible frontend server.
|
||||
@@ -74,11 +38,6 @@ class BaseFrontendArgs:
|
||||
the subclasses.
|
||||
"""
|
||||
|
||||
lora_modules: list[LoRAModulePath] | None = None
|
||||
"""LoRA modules configurations in either 'name=path' format or JSON format
|
||||
or JSON list format. Example (old format): `'name=path'` Example (new
|
||||
format): `{\"name\": \"name\", \"path\": \"lora_path\",
|
||||
\"base_model_name\": \"id\"}`"""
|
||||
chat_template: str | None = None
|
||||
"""The file path to the chat template, or the template in single-line form
|
||||
for the specified model."""
|
||||
@@ -170,18 +129,6 @@ class BaseFrontendArgs:
|
||||
"""
|
||||
log_error_stack: bool = False
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
"""
|
||||
If set to True, only enable the Tokens In<>Out endpoint.
|
||||
This is intended for use in a Disaggregated Everything setup.
|
||||
"""
|
||||
enable_scale_out: bool = False
|
||||
"""
|
||||
If set to True, register the scale-out endpoints (`/inference/v1/generate`
|
||||
and `/inference/v1/abort_requests`) on `vllm serve`. `vllm serve
|
||||
--tokens-only` always registers the endpoints it needs regardless of this
|
||||
flag.
|
||||
"""
|
||||
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
||||
"""Controls the ``system_fingerprint`` field on responses.
|
||||
|
||||
@@ -208,11 +155,6 @@ class BaseFrontendArgs:
|
||||
# Special case: default_chat_template_kwargs needs json.loads type
|
||||
frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
|
||||
|
||||
# Special case: LoRA modules need custom parser action and
|
||||
# optional_type(str)
|
||||
frontend_kwargs["lora_modules"]["type"] = optional_type(str)
|
||||
frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
|
||||
|
||||
# Special case: Tool call parser shows built-in options.
|
||||
valid_tool_parsers = list(ToolParserManager.list_registered())
|
||||
parsers_str = ",".join(valid_tool_parsers)
|
||||
|
||||
@@ -33,7 +33,6 @@ from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
|
||||
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.logger import configure_logging_if_needed, init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.outputs import PoolingRequestOutput, RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
@@ -415,7 +414,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
@@ -439,7 +437,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
@@ -469,7 +466,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
params=sampling_params,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
priority=priority,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
@@ -479,7 +475,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
@@ -494,7 +489,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
Args:
|
||||
prompts: The prompts to the LLM. See generate() for details.
|
||||
sampling_params: The sampling parameters for text generation.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
@@ -519,7 +513,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
prompts=prompts,
|
||||
params=sampling_params,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
@@ -614,7 +607,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
| Sequence[list[ChatCompletionMessageParam]],
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
@@ -649,7 +641,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
@@ -698,7 +689,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
params=sampling_params,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
@@ -716,7 +706,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
| Sequence[list[ChatCompletionMessageParam]],
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
@@ -742,7 +731,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
If None, we use the default sampling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar while rendering
|
||||
conversations.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
@@ -780,7 +768,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
messages=messages,
|
||||
params=sampling_params,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
|
||||
@@ -136,8 +136,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
if raw_request:
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
|
||||
model_name = self.models.model_name(lora_request)
|
||||
model_name = self.models.model_name()
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
@@ -161,7 +160,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
sub_request_id,
|
||||
engine_prompt,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
trace_headers = (
|
||||
None
|
||||
@@ -180,7 +178,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
engine_prompt,
|
||||
sampling_params,
|
||||
sub_request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
|
||||
@@ -265,7 +265,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
"""Validate the model and preprocess a chat completion request.
|
||||
|
||||
Delegates preprocessing logic to OnlineRenderer, adding the
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
engine-aware checks (engine health).
|
||||
|
||||
Returns:
|
||||
A tuple of (conversation, engine_inputs) on success,
|
||||
@@ -322,9 +322,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
if raw_request:
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
|
||||
|
||||
model_name = self.models.model_name(lora_request)
|
||||
model_name = self.models.model_name()
|
||||
|
||||
# Extract data_parallel_rank from header (router can inject it)
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
@@ -369,7 +367,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
sub_request_id,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = (
|
||||
@@ -384,7 +381,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt=engine_input,
|
||||
request_id=sub_request_id,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
)
|
||||
@@ -393,7 +389,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
engine_input,
|
||||
sampling_params,
|
||||
sub_request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=self._get_priority(request, raw_request),
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
|
||||
@@ -94,7 +94,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
"""Validate the model and preprocess a completion request.
|
||||
|
||||
Delegates preprocessing logic to OnlineRenderer, adding the
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
engine-aware checks (engine health).
|
||||
|
||||
Returns:
|
||||
A list of engine_inputs on success, or an ErrorResponse on failure.
|
||||
@@ -147,8 +147,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
if raw_request:
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
# Extract data_parallel_rank from header (router can inject it)
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
|
||||
@@ -182,7 +180,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
request_id_item,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = (
|
||||
@@ -197,7 +194,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
prompt=engine_input,
|
||||
request_id=request_id,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
)
|
||||
@@ -206,7 +202,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=self._get_priority(request, raw_request),
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
@@ -217,7 +212,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
|
||||
result_generator = merge_async_iterators(*generators)
|
||||
|
||||
model_name = self.models.model_name(lora_request)
|
||||
model_name = self.models.model_name()
|
||||
num_prompts = len(engine_inputs)
|
||||
|
||||
# Streaming response
|
||||
|
||||
@@ -9,11 +9,3 @@ from dataclasses import dataclass
|
||||
class BaseModelPath:
|
||||
name: str
|
||||
model_path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoRAModulePath:
|
||||
name: str
|
||||
path: str
|
||||
base_model_name: str | None = None
|
||||
is_3d_lora_weight: bool = False
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from asyncio import Lock
|
||||
from collections import defaultdict
|
||||
from http import HTTPStatus
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
@@ -15,24 +13,12 @@ from vllm.entrypoints.serve.engine.protocol import (
|
||||
ModelList,
|
||||
ModelPermission,
|
||||
)
|
||||
from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
UnloadLoRAAdapterRequest,
|
||||
)
|
||||
from vllm.exceptions import LoRAAdapterNotFoundError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
||||
from vllm.utils.counter import AtomicCounter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIModelRegistry:
|
||||
"""Read-only view of the loaded base models with no engine dependency.
|
||||
|
||||
Suitable for CPU-only / render-only contexts that have no engine client
|
||||
and no LoRA support.
|
||||
Suitable for CPU-only / render-only contexts that have no engine client.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -42,9 +28,8 @@ class OpenAIModelRegistry:
|
||||
) -> None:
|
||||
self.model_config = model_config
|
||||
self.base_model_paths = base_model_paths
|
||||
self.lora_requests: dict[str, LoRARequest] = {}
|
||||
|
||||
def model_name(self, lora_request: LoRARequest | None = None) -> str:
|
||||
def model_name(self) -> str:
|
||||
return self.base_model_paths[0].name
|
||||
|
||||
def is_base_model(self, model_name: str) -> bool:
|
||||
@@ -76,25 +61,18 @@ class OpenAIModelRegistry:
|
||||
]
|
||||
)
|
||||
|
||||
async def resolve_lora(self, lora_name: str):
|
||||
raise RuntimeError("The OpenAIModelRegistry has no LoRA support.")
|
||||
|
||||
|
||||
class OpenAIServingModels:
|
||||
"""Shared instance to hold data about the loaded base model(s) and adapters.
|
||||
"""Shared instance to hold data about the loaded base model(s).
|
||||
|
||||
Handles the routes:
|
||||
- /v1/models
|
||||
- /v1/load_lora_adapter
|
||||
- /v1/unload_lora_adapter
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
base_model_paths: list[BaseModelPath],
|
||||
*,
|
||||
lora_modules: list[LoRAModulePath] | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -106,258 +84,16 @@ class OpenAIServingModels:
|
||||
self.engine_client = engine_client
|
||||
self.base_model_paths = base_model_paths
|
||||
|
||||
self.static_lora_modules = lora_modules
|
||||
self.lora_requests: dict[str, LoRARequest] = {}
|
||||
self.lora_id_counter = AtomicCounter(0)
|
||||
|
||||
self.lora_resolvers: list[LoRAResolver] = []
|
||||
for lora_resolver_name in LoRAResolverRegistry.get_supported_resolvers():
|
||||
self.lora_resolvers.append(
|
||||
LoRAResolverRegistry.get_resolver(lora_resolver_name)
|
||||
)
|
||||
self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)
|
||||
|
||||
self.model_config = self.engine_client.model_config
|
||||
self.renderer = self.engine_client.renderer
|
||||
self.input_processor = self.engine_client.input_processor
|
||||
|
||||
async def init_static_loras(self):
|
||||
"""Loads all static LoRA modules.
|
||||
Raises if any fail to load"""
|
||||
if self.static_lora_modules is None:
|
||||
return
|
||||
for lora in self.static_lora_modules:
|
||||
load_request = LoadLoRAAdapterRequest(
|
||||
lora_path=lora.path,
|
||||
lora_name=lora.name,
|
||||
is_3d_lora_weight=lora.is_3d_lora_weight,
|
||||
)
|
||||
load_result = await self.load_lora_adapter(
|
||||
request=load_request, base_model_name=lora.base_model_name
|
||||
)
|
||||
if isinstance(load_result, ErrorResponse):
|
||||
raise ValueError(load_result.error.message)
|
||||
|
||||
def is_base_model(self, model_name: str) -> bool:
|
||||
return self.registry.is_base_model(model_name)
|
||||
|
||||
def model_name(self, lora_request: LoRARequest | None = None) -> str:
|
||||
if lora_request is not None:
|
||||
return lora_request.lora_name
|
||||
def model_name(self) -> str:
|
||||
return self.base_model_paths[0].name
|
||||
|
||||
async def show_available_models(self) -> ModelList:
|
||||
"""Show available models. This includes the base model and all
|
||||
adapters."""
|
||||
model_list = await self.registry.show_available_models()
|
||||
lora_cards = [
|
||||
ModelCard(
|
||||
id=lora.lora_name,
|
||||
root=lora.path,
|
||||
parent=lora.base_model_name
|
||||
if lora.base_model_name
|
||||
else self.base_model_paths[0].name,
|
||||
permission=[ModelPermission()],
|
||||
)
|
||||
for lora in self.lora_requests.values()
|
||||
]
|
||||
model_list.data.extend(lora_cards)
|
||||
return model_list
|
||||
|
||||
async def load_lora_adapter(
|
||||
self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
|
||||
) -> ErrorResponse | str:
|
||||
lora_name = request.lora_name
|
||||
|
||||
# Ensure atomicity based on the lora name
|
||||
async with self.lora_resolver_lock[lora_name]:
|
||||
error_check_ret = await self._check_load_lora_adapter_request(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
lora_path = request.lora_path
|
||||
lora_int_id = (
|
||||
self.lora_requests[lora_name].lora_int_id
|
||||
if lora_name in self.lora_requests
|
||||
else self.lora_id_counter.inc(1)
|
||||
)
|
||||
lora_request = LoRARequest(
|
||||
lora_name=lora_name,
|
||||
lora_int_id=lora_int_id,
|
||||
lora_path=lora_path,
|
||||
load_inplace=request.load_inplace,
|
||||
is_3d_lora_weight=request.is_3d_lora_weight,
|
||||
)
|
||||
if base_model_name is not None and self.is_base_model(base_model_name):
|
||||
lora_request.base_model_name = base_model_name
|
||||
|
||||
# Validate that the adapter can be loaded into the engine
|
||||
# This will also preload it for incoming requests
|
||||
try:
|
||||
await self.engine_client.add_lora(lora_request)
|
||||
except Exception as e:
|
||||
if str(
|
||||
LoRAAdapterNotFoundError(
|
||||
lora_request.lora_name, lora_request.lora_path
|
||||
)
|
||||
) in str(e):
|
||||
return create_error_response(
|
||||
LoRAAdapterNotFoundError(
|
||||
lora_request.lora_name, lora_request.lora_path
|
||||
)
|
||||
)
|
||||
return create_error_response(
|
||||
message=str(e),
|
||||
err_type="InternalServerError",
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
previous = self.lora_requests.get(lora_name)
|
||||
if previous is not None and previous.lora_path == lora_path:
|
||||
logger.warning(
|
||||
"Reloaded LoRA adapter '%s' in place from the same path '%s'. "
|
||||
"Prefix-cache blocks computed with its previous weights are "
|
||||
"not invalidated; load changed weights from a new path to "
|
||||
"avoid reusing them.",
|
||||
lora_name,
|
||||
lora_path,
|
||||
)
|
||||
self.lora_requests[lora_name] = lora_request
|
||||
logger.info(
|
||||
"Loaded new LoRA adapter: name '%s', path '%s'", lora_name, lora_path
|
||||
)
|
||||
return f"Success: LoRA adapter '{lora_name}' added successfully."
|
||||
|
||||
async def unload_lora_adapter(
|
||||
self, request: UnloadLoRAAdapterRequest
|
||||
) -> ErrorResponse | str:
|
||||
lora_name = request.lora_name
|
||||
|
||||
# Ensure atomicity based on the lora name
|
||||
async with self.lora_resolver_lock[lora_name]:
|
||||
error_check_ret = await self._check_unload_lora_adapter_request(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
# Safe to delete now since we hold the lock
|
||||
del self.lora_requests[lora_name]
|
||||
logger.info("Removed LoRA adapter: name '%s'", lora_name)
|
||||
return f"Success: LoRA adapter '{lora_name}' removed successfully."
|
||||
|
||||
async def _check_load_lora_adapter_request(
|
||||
self, request: LoadLoRAAdapterRequest
|
||||
) -> ErrorResponse | None:
|
||||
# Check if both 'lora_name' and 'lora_path' are provided
|
||||
if not request.lora_name or not request.lora_path:
|
||||
return create_error_response(
|
||||
message="Both 'lora_name' and 'lora_path' must be provided.",
|
||||
err_type="InvalidUserInput",
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
if self.is_base_model(request.lora_name):
|
||||
return create_error_response(
|
||||
message=f"The lora adapter '{request.lora_name}' conflicts with a "
|
||||
"served base model.",
|
||||
err_type="InvalidUserInput",
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
# If not loading inplace
|
||||
# Check if the lora adapter with the given name already exists
|
||||
if not request.load_inplace and request.lora_name in self.lora_requests:
|
||||
return create_error_response(
|
||||
message=f"The lora adapter '{request.lora_name}' has already been "
|
||||
"loaded. If you want to load the adapter in place, set 'load_inplace'"
|
||||
" to True.",
|
||||
err_type="InvalidUserInput",
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
async def _check_unload_lora_adapter_request(
|
||||
self, request: UnloadLoRAAdapterRequest
|
||||
) -> ErrorResponse | None:
|
||||
# Check if 'lora_name' is not provided return an error
|
||||
if not request.lora_name:
|
||||
return create_error_response(
|
||||
message="'lora_name' needs to be provided to unload a LoRA adapter.",
|
||||
err_type="InvalidUserInput",
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
# Check if the lora adapter with the given name exists
|
||||
if request.lora_name not in self.lora_requests:
|
||||
return create_error_response(
|
||||
message=f"The lora adapter '{request.lora_name}' cannot be found.",
|
||||
err_type="NotFoundError",
|
||||
status_code=HTTPStatus.NOT_FOUND,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
async def resolve_lora(self, lora_name: str) -> LoRARequest | ErrorResponse:
|
||||
"""Attempt to resolve a LoRA adapter using available resolvers.
|
||||
|
||||
Args:
|
||||
lora_name: Name/identifier of the LoRA adapter
|
||||
|
||||
Returns:
|
||||
LoRARequest if found and loaded successfully.
|
||||
ErrorResponse (404) if no resolver finds the adapter.
|
||||
ErrorResponse (400) if adapter(s) are found but none load.
|
||||
|
||||
"""
|
||||
async with self.lora_resolver_lock[lora_name]:
|
||||
# First check if this LoRA is already loaded
|
||||
if lora_name in self.lora_requests:
|
||||
return self.lora_requests[lora_name]
|
||||
|
||||
base_model_name = self.model_config.model
|
||||
unique_id = self.lora_id_counter.inc(1)
|
||||
found_adapter = False
|
||||
|
||||
# Try to resolve using available resolvers
|
||||
for resolver in self.lora_resolvers:
|
||||
lora_request = await resolver.resolve_lora(base_model_name, lora_name)
|
||||
|
||||
if lora_request is not None:
|
||||
found_adapter = True
|
||||
lora_request.lora_int_id = unique_id
|
||||
|
||||
try:
|
||||
await self.engine_client.add_lora(lora_request)
|
||||
self.lora_requests[lora_name] = lora_request
|
||||
logger.info(
|
||||
"Resolved and loaded LoRA adapter '%s' using %s",
|
||||
lora_name,
|
||||
resolver.__class__.__name__,
|
||||
)
|
||||
return lora_request
|
||||
except BaseException as e:
|
||||
logger.warning(
|
||||
"Failed to load LoRA '%s' resolved by %s: %s. "
|
||||
"Trying next resolver.",
|
||||
lora_name,
|
||||
resolver.__class__.__name__,
|
||||
e,
|
||||
)
|
||||
continue
|
||||
|
||||
if found_adapter:
|
||||
# An adapter was found, but all attempts to load it failed.
|
||||
return create_error_response(
|
||||
message=(
|
||||
f"LoRA adapter '{lora_name}' was found but could not be loaded."
|
||||
),
|
||||
err_type="BadRequestError",
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
else:
|
||||
# No adapter was found
|
||||
return create_error_response(
|
||||
message=f"LoRA adapter {lora_name} does not exist",
|
||||
err_type="NotFoundError",
|
||||
status_code=HTTPStatus.NOT_FOUND,
|
||||
)
|
||||
"""Show available models."""
|
||||
return await self.registry.show_available_models()
|
||||
|
||||
@@ -81,7 +81,6 @@ from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import TokenizeParams
|
||||
@@ -370,8 +369,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
else:
|
||||
prev_response = None
|
||||
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
model_name = self.models.model_name(lora_request)
|
||||
model_name = self.models.model_name()
|
||||
|
||||
render_result = await self._render_resolved_response_inputs(
|
||||
request,
|
||||
@@ -521,7 +519,6 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
if isinstance(context, HarmonyContext)
|
||||
else None
|
||||
),
|
||||
lora_request=lora_request,
|
||||
priority=self._get_priority(request, raw_request),
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
@@ -643,7 +640,6 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
sampling_params: SamplingParams,
|
||||
context: ConversationContext,
|
||||
tok_params: TokenizeParams | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
session_id: str | None = None,
|
||||
@@ -663,14 +659,12 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
sub_request_id,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
sub_request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
session_id=session_id,
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
else:
|
||||
RequestLogger = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def init_scale_out_state(
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
engine_client: "EngineClient",
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
|
||||
state.serving_tokens = ServingTokens(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
force_no_detokenize=args.tokens_only,
|
||||
)
|
||||
|
||||
|
||||
def register_scale_out_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
args = getattr(app.state, "args", None)
|
||||
# `vllm serve --tokens-only` is dedicated to serving these endpoints, so it
|
||||
# always registers them regardless of `--enable-scale-out`.
|
||||
dedicated_mode = getattr(args, "tokens_only", False)
|
||||
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
|
||||
if not enabled:
|
||||
message = (
|
||||
"Scale-out endpoints are disabled. Restart vllm serve with "
|
||||
"--enable-scale-out to enable them."
|
||||
)
|
||||
logger.info(message)
|
||||
|
||||
async def disabled_scale_out() -> JSONResponse:
|
||||
error = create_error_response(
|
||||
message, "PermissionDeniedError", HTTPStatus.FORBIDDEN
|
||||
)
|
||||
return JSONResponse(error.model_dump(), status_code=HTTPStatus.FORBIDDEN)
|
||||
|
||||
from .token_in_token_out.api_router import router as token_router
|
||||
|
||||
for route in token_router.routes:
|
||||
if isinstance(route, APIRoute):
|
||||
app.add_api_route(
|
||||
route.path,
|
||||
disabled_scale_out,
|
||||
methods=sorted(route.methods),
|
||||
include_in_schema=False,
|
||||
)
|
||||
return
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from .token_in_token_out.api_router import (
|
||||
attach_router as attach_disagg_router,
|
||||
)
|
||||
|
||||
attach_disagg_router(app)
|
||||
@@ -1,117 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from .protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponseBase,
|
||||
)
|
||||
from .serving import ServingTokens
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def tokenization(request: Request) -> ServingTokenization:
|
||||
return request.app.state.serving_tokenization
|
||||
|
||||
|
||||
def generate_tokens(request: Request) -> ServingTokens | None:
|
||||
return request.app.state.serving_tokens
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/inference/v1/generate",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def generate(request: GenerateRequest, raw_request: Request):
|
||||
handler = generate_tokens(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support generate tokens API")
|
||||
|
||||
generator = await handler.serve_tokens(request, raw_request)
|
||||
|
||||
if isinstance(generator, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=generator.model_dump(), status_code=generator.error.code
|
||||
)
|
||||
|
||||
elif isinstance(generator, GenerateResponseBase):
|
||||
return JSONResponse(content=generator.model_dump())
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
|
||||
|
||||
abort_router = APIRouter()
|
||||
|
||||
|
||||
async def abort_requests(raw_request: Request):
|
||||
"""Abort one or more requests. To be used in a
|
||||
Disaggregated Everything setup.
|
||||
"""
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
request_ids = body.get("request_ids")
|
||||
if request_ids is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Missing 'request_ids' in request body",
|
||||
)
|
||||
# Abort requests in background
|
||||
asyncio.create_task(engine_client(raw_request).abort(request_ids))
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
# Under the `/inference` prefix, so `--api-key` guards it.
|
||||
router.add_api_route("/inference/v1/abort_requests", abort_requests, methods=["POST"])
|
||||
# Unauthenticated legacy path, kept for existing `--tokens-only` deployments.
|
||||
abort_router.add_api_route("/abort_requests", abort_requests, methods=["POST"])
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
if not getattr(app.state.args, "tokens_only", False):
|
||||
return
|
||||
# An endpoint plugin may already serve /abort_requests. Registering this
|
||||
# one too would only add a shadowed route with a duplicate operation ID.
|
||||
has_abort_route = any(
|
||||
getattr(route, "path", None) == "/abort_requests" for route in app.routes
|
||||
)
|
||||
if not has_abort_route:
|
||||
app.include_router(abort_router)
|
||||
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helpers for render-time multimodal feature extraction and generate input."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Collection, Sequence
|
||||
from typing import cast
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import MMPayload
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||
decode_mm_kwargs_item,
|
||||
encode_mm_kwargs_item,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.inputs import (
|
||||
EngineInput,
|
||||
MultiModalHashes,
|
||||
MultiModalInput,
|
||||
MultiModalPlaceholders,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsOptionalItems,
|
||||
)
|
||||
|
||||
|
||||
def merge_mm_kwargs_items(
|
||||
kwargs_item: MultiModalKwargsItem | None,
|
||||
metadata_item: MultiModalKwargsItem | None,
|
||||
) -> MultiModalKwargsItem | None:
|
||||
"""Merge full multimodal data with its metadata-only counterpart."""
|
||||
if kwargs_item is None:
|
||||
return metadata_item
|
||||
if metadata_item is None:
|
||||
return kwargs_item
|
||||
return MultiModalKwargsItem({**kwargs_item, **metadata_item})
|
||||
|
||||
|
||||
def _encode_metadata_items(
|
||||
items: Sequence[MultiModalKwargsItem | None],
|
||||
*,
|
||||
declared: Collection[str],
|
||||
) -> list[str | None]:
|
||||
"""Serialize placeholder-metadata and ``keep_on_cpu`` fields per item."""
|
||||
metadata_items: list[str | None] = []
|
||||
for item in items:
|
||||
if item is None:
|
||||
metadata_items.append(None)
|
||||
continue
|
||||
|
||||
metadata_item = MultiModalKwargsItem(
|
||||
{
|
||||
key: elem
|
||||
for key, elem in item.items()
|
||||
if elem.field.keep_on_cpu or key in declared
|
||||
}
|
||||
)
|
||||
metadata_items.append(
|
||||
encode_mm_kwargs_item(metadata_item) if metadata_item else None
|
||||
)
|
||||
return metadata_items
|
||||
|
||||
|
||||
def _encode_mm_kwargs_with_metadata(
|
||||
raw_mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
*,
|
||||
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
|
||||
) -> tuple[dict[str, list[str | None]], dict[str, list[str | None]] | None]:
|
||||
"""Serialize full kwargs and their metadata-only subsets per modality."""
|
||||
kwargs_data: dict[str, list[str | None]] = {}
|
||||
metadata_by_modality: dict[str, list[str | None]] = {}
|
||||
|
||||
for modality, items in raw_mm_kwargs.items():
|
||||
kwargs_data[modality] = [
|
||||
encode_mm_kwargs_item(item) if item is not None else None for item in items
|
||||
]
|
||||
|
||||
declared = (
|
||||
set(metadata_fields_for(modality))
|
||||
if metadata_fields_for is not None
|
||||
else set()
|
||||
)
|
||||
metadata_items = _encode_metadata_items(items, declared=declared)
|
||||
if any(item is not None for item in metadata_items):
|
||||
metadata_by_modality[modality] = metadata_items
|
||||
|
||||
mm_metadata = metadata_by_modality or None
|
||||
return kwargs_data, mm_metadata
|
||||
|
||||
|
||||
def mm_kwargs_from_features(
|
||||
features: MultiModalFeatures,
|
||||
) -> dict[str, list[MultiModalKwargsItem | None]]:
|
||||
"""Deserialize ``features`` into per-modality kwargs for ``mm_input``."""
|
||||
mm_kwargs: dict[str, list[MultiModalKwargsItem | None]] = {}
|
||||
kwargs_data = features.kwargs_data or {}
|
||||
mm_metadata = features.mm_metadata or {}
|
||||
for modality, hashes in features.mm_hashes.items():
|
||||
n = len(hashes)
|
||||
kwargs_items = [
|
||||
decode_mm_kwargs_item(item) if item is not None else None
|
||||
for item in kwargs_data.get(modality, [None] * n)
|
||||
]
|
||||
metadata_items = [
|
||||
decode_mm_kwargs_item(item) if item is not None else None
|
||||
for item in mm_metadata.get(modality, [None] * n)
|
||||
]
|
||||
mm_kwargs[modality] = [
|
||||
merge_mm_kwargs_items(kwargs_item, metadata_item)
|
||||
for kwargs_item, metadata_item in zip(
|
||||
kwargs_items, metadata_items, strict=True
|
||||
)
|
||||
]
|
||||
return mm_kwargs
|
||||
|
||||
|
||||
def placeholder_ranges_from_engine_input(
|
||||
engine_input: EngineInput,
|
||||
) -> dict[str, list[PlaceholderRangeInfo]] | None:
|
||||
"""Return per-modality placeholder ranges, or ``None`` for text-only prompts."""
|
||||
if engine_input.get("type") != "multimodal":
|
||||
return None
|
||||
|
||||
raw_placeholders: MultiModalPlaceholders = cast(MultiModalInput, engine_input)[
|
||||
"mm_placeholders"
|
||||
]
|
||||
return {
|
||||
modality: [
|
||||
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in raw_placeholders.items()
|
||||
}
|
||||
|
||||
|
||||
def extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
*,
|
||||
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
|
||||
mm_payload: MMPayload = "tensors",
|
||||
) -> MultiModalFeatures | None:
|
||||
"""Extract multimodal features from a rendered engine prompt.
|
||||
|
||||
Returns ``None`` for text-only prompts. ``mm_metadata`` keeps the
|
||||
intersection of processed kwargs that prefill needs after EC transfer:
|
||||
fields declared as embedding metadata, plus fields marked
|
||||
``keep_on_cpu`` (for example M-RoPE grid dims). With
|
||||
``mm_payload="none"`` neither is serialized.
|
||||
"""
|
||||
if engine_input.get("type") != "multimodal":
|
||||
return None
|
||||
|
||||
mm_engine_input = cast(MultiModalInput, engine_input)
|
||||
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
|
||||
mm_placeholders = placeholder_ranges_from_engine_input(engine_input)
|
||||
assert mm_placeholders is not None
|
||||
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
mm_metadata: dict[str, list[str | None]] | None = None
|
||||
if mm_payload == "tensors":
|
||||
raw_mm_kwargs = mm_engine_input.get("mm_kwargs")
|
||||
if raw_mm_kwargs:
|
||||
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
|
||||
raw_mm_kwargs,
|
||||
metadata_fields_for=metadata_fields_for,
|
||||
)
|
||||
|
||||
return MultiModalFeatures(
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
kwargs_data=kwargs_data,
|
||||
mm_metadata=mm_metadata,
|
||||
)
|
||||
@@ -1,27 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Encode/decode utilities for multimodal tensors and field metadata
|
||||
over JSON/HTTP, used by the disaggregated generate endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pybase64
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
_encoder = MsgpackEncoder(size_threshold=2**62) # force all tensors inline
|
||||
_decoder = MsgpackDecoder(t=MultiModalKwargsItem)
|
||||
|
||||
|
||||
def encode_mm_kwargs_item(item: MultiModalKwargsItem) -> str:
|
||||
"""Serialize a MultiModalKwargsItem to a base64 string."""
|
||||
bufs = _encoder.encode(item)
|
||||
assert len(bufs) == 1, "All tensors should be inline"
|
||||
return pybase64.b64encode(bufs[0]).decode("ascii")
|
||||
|
||||
|
||||
def decode_mm_kwargs_item(data: str) -> MultiModalKwargsItem:
|
||||
"""Deserialize a base64 string back to a MultiModalKwargsItem."""
|
||||
raw = pybase64.b64decode(data)
|
||||
return _decoder.decode(raw)
|
||||
@@ -1,904 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Discriminator,
|
||||
Field,
|
||||
NonNegativeInt,
|
||||
PrivateAttr,
|
||||
Tag,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
StreamOptions,
|
||||
validate_cache_salt,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import WeightVersionSpan
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
####### Tokens IN <> Tokens OUT #######
|
||||
|
||||
OutputMode: TypeAlias = Literal["tokens", "text"]
|
||||
|
||||
|
||||
class PlaceholderRangeInfo(BaseModel):
|
||||
"""Serializable placeholder location for a single multi-modal item."""
|
||||
|
||||
offset: int = Field(ge=0)
|
||||
"""Start index of the placeholder tokens in the prompt."""
|
||||
|
||||
length: int = Field(gt=0)
|
||||
"""Number of placeholder tokens."""
|
||||
|
||||
# TODO: add `is_embed: list[bool] | None` once the /generate side
|
||||
# consumes features — some models (e.g. Qwen-VL) use sparse
|
||||
# placeholder masks that cannot be recomputed from offset+length alone.
|
||||
|
||||
|
||||
def _has_serialized_mm_items(
|
||||
payload: dict[str, list[str | None]] | None,
|
||||
) -> bool:
|
||||
if not payload:
|
||||
return False
|
||||
return any(item is not None for items in payload.values() for item in items)
|
||||
|
||||
|
||||
class MultiModalFeatures(BaseModel):
|
||||
"""Lightweight multimodal metadata produced by the render step.
|
||||
|
||||
Carries hashes (for cache lookup / identification) and placeholder
|
||||
positions so the downstream `/generate` service knows *where* in
|
||||
the token sequence each multimodal item lives.
|
||||
"""
|
||||
|
||||
mm_hashes: dict[str, list[str]]
|
||||
"""Per-modality item hashes, e.g. `{"image": ["abc", "def"]}`."""
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
|
||||
"""Per-modality placeholder ranges in the token sequence."""
|
||||
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
"""Per-modality serialized tensor data.
|
||||
|
||||
Each value is a list parallel to `mm_hashes[modality]`. A `str`
|
||||
entry is a base64-encoded `MultiModalKwargsItem`; `None` means
|
||||
the item should be resolved from cache. The entire field is
|
||||
`None` for metadata-only (cache-hit) responses.
|
||||
"""
|
||||
|
||||
mm_metadata: dict[str, list[str | None]] | None = None
|
||||
"""Per-modality serialized metadata for disaggregated prefill.
|
||||
|
||||
Each value is a list parallel to `mm_hashes[modality]`. A `str`
|
||||
entry is a base64-encoded `MultiModalKwargsItem` containing only
|
||||
placeholder-metadata and `keep_on_cpu` fields. `None` means that
|
||||
the metadata is unavailable for that item. Prefill can use this
|
||||
instead of `kwargs_data` only when `ec_transfer_params` is also
|
||||
set, so embeddings arrive through the EC connector rather than from
|
||||
`pixel_values`.
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_parallel_fields(self) -> "MultiModalFeatures":
|
||||
modalities = set(self.mm_hashes)
|
||||
if set(self.mm_placeholders) != modalities:
|
||||
raise ValueError(
|
||||
"mm_hashes and mm_placeholders must use the same modalities"
|
||||
)
|
||||
if self.kwargs_data is not None and set(self.kwargs_data) != modalities:
|
||||
raise ValueError("kwargs_data must use the same modalities as mm_hashes")
|
||||
if self.mm_metadata is not None and set(self.mm_metadata) != modalities:
|
||||
raise ValueError("mm_metadata must use the same modalities as mm_hashes")
|
||||
|
||||
flattened_ranges: list[tuple[int, int]] = []
|
||||
for modality in modalities:
|
||||
num_hashes = len(self.mm_hashes[modality])
|
||||
num_placeholders = len(self.mm_placeholders[modality])
|
||||
if num_hashes != num_placeholders:
|
||||
raise ValueError(
|
||||
f"{modality} mm_hashes and mm_placeholders must have "
|
||||
"the same length"
|
||||
)
|
||||
if (
|
||||
self.kwargs_data is not None
|
||||
and len(self.kwargs_data[modality]) != num_hashes
|
||||
):
|
||||
raise ValueError(
|
||||
f"{modality} kwargs_data and mm_hashes must have the same length"
|
||||
)
|
||||
if (
|
||||
self.mm_metadata is not None
|
||||
and len(self.mm_metadata[modality]) != num_hashes
|
||||
):
|
||||
raise ValueError(
|
||||
f"{modality} mm_metadata and mm_hashes must have the same length"
|
||||
)
|
||||
flattened_ranges.extend(
|
||||
(placeholder.offset, placeholder.offset + placeholder.length)
|
||||
for placeholder in self.mm_placeholders[modality]
|
||||
)
|
||||
|
||||
flattened_ranges.sort()
|
||||
for (offset, end), (next_offset, _) in zip(
|
||||
flattened_ranges, flattened_ranges[1:]
|
||||
):
|
||||
if next_offset < end:
|
||||
raise ValueError(
|
||||
"mm_placeholders must be globally non-overlapping and sorted"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class ReasoningParserKwargs(BaseModel):
|
||||
"""Kwargs for the engine-side reasoning parser that gates structured
|
||||
outputs. Typed so clients cannot pass arbitrary constructor kwargs."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = Field(default_factory=dict)
|
||||
"""The effective chat template kwargs the prompt was rendered with."""
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
token_ids: list[int] = Field(min_length=1)
|
||||
"""The token ids to generate text from."""
|
||||
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
def validate_token_ids(cls, v: list[int]) -> list[int]:
|
||||
if any(t < 0 for t in v):
|
||||
raise ValueError("token_ids must not contain negative values")
|
||||
return v
|
||||
|
||||
token_offsets: list[tuple[int, int]] | None = None
|
||||
"""Char-level (start, end) offsets per token, relative to the
|
||||
tokenized source string. Present only when the request set
|
||||
`return_token_offsets=True` and the renderer was able to compute
|
||||
them (Fast tokenizer, text input, no multimodal data). List length
|
||||
equals `token_ids` length when present. None otherwise."""
|
||||
|
||||
features: MultiModalFeatures | None = None
|
||||
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
|
||||
|
||||
content_parts: list[dict[str, Any]] | None = None
|
||||
"""Raw multimodal input; server resolves media. Mutually exclusive
|
||||
with `features`."""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_mm_fields_exclusive(self) -> "GenerateRequest":
|
||||
if self.content_parts and self.features:
|
||||
raise ValueError("content_parts and features are mutually exclusive")
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_ec_for_metadata_only(self) -> "GenerateRequest":
|
||||
features = self.features
|
||||
if features is None:
|
||||
return self
|
||||
if not _has_serialized_mm_items(features.mm_metadata):
|
||||
return self
|
||||
if self.ec_transfer_params:
|
||||
return self
|
||||
kwargs_data = features.kwargs_data or {}
|
||||
for modality, metadata_items in (features.mm_metadata or {}).items():
|
||||
kwargs_items = kwargs_data.get(modality, [None] * len(metadata_items))
|
||||
if any(
|
||||
metadata is not None and kwargs is None
|
||||
for metadata, kwargs in zip(metadata_items, kwargs_items, strict=True)
|
||||
):
|
||||
raise ValueError(
|
||||
"metadata-only multimodal items require ec_transfer_params"
|
||||
)
|
||||
return self
|
||||
|
||||
sampling_params: SamplingParams
|
||||
"""The sampling parameters for the model."""
|
||||
|
||||
model: str | None = None
|
||||
|
||||
reasoning_ended: bool | None = None
|
||||
"""Whether reasoning has ended before the first generated token, as
|
||||
resolved by /render. `True` applies structured outputs from the first
|
||||
token; `None` lets the engine check the prompt with its reasoning parser."""
|
||||
|
||||
reasoning_parser_kwargs: ReasoningParserKwargs | None = None
|
||||
"""Set by /render when it has a reasoning parser, so the engine-side
|
||||
parser agrees with the frontend on flags such as `enable_thinking`."""
|
||||
|
||||
return_token_ids: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If true, return the final prompt token IDs after multimodal "
|
||||
"placeholder expansion, together with multimodal placeholder ranges. "
|
||||
"In streaming mode, this metadata is included only in the first chunk."
|
||||
),
|
||||
)
|
||||
|
||||
output_mode: OutputMode = Field(
|
||||
default="tokens",
|
||||
description=(
|
||||
"Response level. 'tokens' returns token IDs only. 'text' also "
|
||||
"returns the detokenized text and logprobs with decoded token "
|
||||
"strings. 'text' needs a server that loads a tokenizer and "
|
||||
"sampling_params.detokenize to be true."
|
||||
),
|
||||
)
|
||||
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(
|
||||
default=0,
|
||||
ge=-(2**63),
|
||||
le=2**63 - 1,
|
||||
description=(
|
||||
"The priority of the request (lower means earlier handling; "
|
||||
"default: 0). Any priority other than 0 will raise an error "
|
||||
"if the served model does not use priority scheduling."
|
||||
),
|
||||
)
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
|
||||
# Tracks which keys the caller explicitly set inside `sampling_params`
|
||||
# when the request was parsed from a JSON body. Lets the server tell
|
||||
# "client said max_tokens=16" from "client said nothing → dataclass
|
||||
# default 16" so it can apply server-side defaulting only in the latter
|
||||
# case. `None` means the request was constructed with a pre-built
|
||||
# `SamplingParams` instance (e.g. from internal callers that have
|
||||
# already resolved values), in which case all fields are considered set.
|
||||
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
|
||||
_response_mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = (
|
||||
PrivateAttr(default=None)
|
||||
)
|
||||
|
||||
@model_validator(mode="wrap")
|
||||
@classmethod
|
||||
def _capture_sampling_params_provided_keys(cls, data: Any, handler):
|
||||
provided: set[str] | None = None
|
||||
if isinstance(data, dict):
|
||||
sp = data.get("sampling_params")
|
||||
if isinstance(sp, dict):
|
||||
provided = set(sp.keys())
|
||||
instance = handler(data)
|
||||
instance._sampling_params_provided_keys = provided
|
||||
return instance
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _validate_cache_salt(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_output_mode_detokenizes(self) -> "GenerateRequest":
|
||||
if self.output_mode != "tokens" and not self.sampling_params.detokenize:
|
||||
raise ValueError(
|
||||
f"output_mode={self.output_mode!r} requires "
|
||||
"sampling_params.detokenize to be true"
|
||||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_multimodal_feature_bounds(self) -> "GenerateRequest":
|
||||
if self.features is None:
|
||||
return self
|
||||
|
||||
prompt_len = len(self.token_ids)
|
||||
for ranges in self.features.mm_placeholders.values():
|
||||
for placeholder in ranges:
|
||||
if placeholder.offset + placeholder.length > prompt_len:
|
||||
raise ValueError(
|
||||
"mm_placeholders must remain within the token_ids sequence"
|
||||
)
|
||||
return self
|
||||
|
||||
def is_sampling_param_provided(self, name: str) -> bool:
|
||||
"""Whether the caller explicitly set `sampling_params.<name>`.
|
||||
|
||||
For requests parsed from a JSON body, this reflects the raw input
|
||||
dict. For requests constructed with a pre-built `SamplingParams`
|
||||
instance, all fields are considered provided so server-side defaults
|
||||
do not clobber values already resolved upstream.
|
||||
"""
|
||||
if self._sampling_params_provided_keys is None:
|
||||
return True
|
||||
return name in self._sampling_params_provided_keys
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=None,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
|
||||
class GenerateLogProb(BaseModel):
|
||||
"""A single (token, logprob) candidate on the generate wire protocol.
|
||||
|
||||
Unlike the OpenAI logprob shapes this carries the integer token id: the
|
||||
generate server has no tokenizer, so decoding to a string belongs in
|
||||
derender (or the coupled chat/completions path), not here.
|
||||
"""
|
||||
|
||||
token_id: int
|
||||
logprob: float
|
||||
rank: int | None = None
|
||||
|
||||
|
||||
class GenerateLogProbsContent(GenerateLogProb):
|
||||
"""The sampled token at one position, plus its top-k candidates.
|
||||
|
||||
``top_logprobs`` is a list, not a dict: JSON turns dict keys into strings
|
||||
and the order would be implicit. It is in the engine's order: the sampled
|
||||
token first, then the remaining candidates in rank order.
|
||||
"""
|
||||
|
||||
top_logprobs: list[GenerateLogProb] = []
|
||||
|
||||
|
||||
class GenerateLogProbs(BaseModel):
|
||||
"""Output logprobs for one choice.
|
||||
|
||||
``content`` holds one entry per generated token.
|
||||
"""
|
||||
|
||||
content: list[GenerateLogProbsContent] | None = None
|
||||
|
||||
|
||||
class GenerateChoiceBase(BaseModel):
|
||||
"""Fields shared by every `output_mode` of a non-streaming choice."""
|
||||
|
||||
index: int
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
# Stop string or token id that ended generation (repetition detection
|
||||
# sets it too); None on EOS, length and abort.
|
||||
stop_reason: int | str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded `.npy` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
|
||||
# `num_tokens - 1` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
# np.load(io.BytesIO(base64.b64decode(s)))
|
||||
# `None` if (a) the request was aborted before any forward pass,
|
||||
# or (b) `enable_return_routed_experts` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
sampling_mask: list[list[int]] | None = None
|
||||
# Contiguous weight version spans over output-token indexes.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
def validate_token_ids(cls, v: list[int] | None) -> list[int] | None:
|
||||
if v is not None and any(t < 0 for t in v):
|
||||
raise ValueError("token_ids must not contain negative values")
|
||||
return v
|
||||
|
||||
|
||||
class GenerateTokensChoice(GenerateChoiceBase):
|
||||
logprobs: GenerateLogProbs | None = None
|
||||
"""Logprobs with integer token ids (no tokenizer on the generate server)."""
|
||||
|
||||
|
||||
class GenerateTextChoice(GenerateChoiceBase):
|
||||
text: str
|
||||
"""Detokenized output. Excludes a matched stop string unless
|
||||
`include_stop_str_in_output` is set, while `token_ids` keeps every
|
||||
generated token."""
|
||||
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
"""Logprobs with decoded token strings and `bytes`."""
|
||||
|
||||
|
||||
class GenerateStreamChoiceBase(BaseModel):
|
||||
"""Fields shared by every `output_mode` of a streaming choice."""
|
||||
|
||||
index: int
|
||||
finish_reason: str | None = None
|
||||
# Stop string or token id that ended generation (repetition detection
|
||||
# sets it too); None on EOS, length and abort.
|
||||
stop_reason: int | str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
routed_experts: str | None = None
|
||||
sampling_mask: list[list[int]] | None = None
|
||||
# Set on the final chunk only; covers the whole output.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
|
||||
logprobs: GenerateLogProbs | None = None
|
||||
|
||||
|
||||
class GenerateTextStreamChoice(GenerateStreamChoiceBase):
|
||||
text: str
|
||||
"""Text delta since the previous chunk of this choice."""
|
||||
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
|
||||
|
||||
class GenerateStreamResponseBase(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class GenerateTokensStreamResponse(GenerateStreamResponseBase):
|
||||
output_mode: Literal["tokens"] = "tokens"
|
||||
choices: list[GenerateTokensStreamChoice]
|
||||
|
||||
|
||||
class GenerateTextStreamResponse(GenerateStreamResponseBase):
|
||||
output_mode: Literal["text"] = "text"
|
||||
choices: list[GenerateTextStreamChoice]
|
||||
|
||||
|
||||
class GenerateResponseBase(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
model: str | None = None
|
||||
created: int | None = None
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_id_logprobs: str | None = None
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class GenerateTokensResponse(GenerateResponseBase):
|
||||
output_mode: Literal["tokens"] = "tokens"
|
||||
choices: list[GenerateTokensChoice]
|
||||
|
||||
|
||||
class GenerateTextResponse(GenerateResponseBase):
|
||||
output_mode: Literal["text"] = "text"
|
||||
choices: list[GenerateTextChoice]
|
||||
|
||||
|
||||
def output_mode_or_tokens(value: Any) -> Any:
|
||||
"""Discriminator for `GenerateResponse` and `GenerateStreamResponse`.
|
||||
|
||||
A missing `output_mode` means `tokens`. Existing clients and servers
|
||||
that predate the field never send it.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
return value.get("output_mode", "tokens")
|
||||
return getattr(value, "output_mode", "tokens")
|
||||
|
||||
|
||||
# Type aliases, not classes: use `GenerateResponseBase` or
|
||||
# `GenerateStreamResponseBase` for `isinstance` checks and the
|
||||
# `Generate<Level>...` classes to construct.
|
||||
GenerateResponse: TypeAlias = Annotated[
|
||||
Annotated[GenerateTokensResponse, Tag("tokens")]
|
||||
| Annotated[GenerateTextResponse, Tag("text")],
|
||||
Discriminator(output_mode_or_tokens),
|
||||
]
|
||||
GenerateStreamResponse: TypeAlias = Annotated[
|
||||
Annotated[GenerateTokensStreamResponse, Tag("tokens")]
|
||||
| Annotated[GenerateTextStreamResponse, Tag("text")],
|
||||
Discriminator(output_mode_or_tokens),
|
||||
]
|
||||
|
||||
|
||||
class DerenderChatRequest(BaseModel):
|
||||
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
|
||||
|
||||
Wraps a complete GenerateResponse and caller supplied metadata needed to
|
||||
produce a fully formed ChatCompletionResponse without a GPU.
|
||||
"""
|
||||
|
||||
# --8<-- [start:derender-chat-request]
|
||||
stream: Literal[False] = False
|
||||
|
||||
model: str | None = None
|
||||
"""Served model name. Defaults to the server's served model name."""
|
||||
|
||||
generate_response: GenerateTokensResponse
|
||||
"""The complete token-in / token-out engine response to derender.
|
||||
|
||||
Only `output_mode="tokens"` responses are accepted. Other modes are
|
||||
already detokenized and re-decoding their `token_ids` would undo the
|
||||
engine's stop string truncation.
|
||||
"""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage; defaults to 0 if omitted.
|
||||
|
||||
GenerateResponse carries only output tokens; the caller already has
|
||||
len(GenerateRequest.token_ids) from the render step.
|
||||
"""
|
||||
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
"""Prompt token IDs (`GenerateRequest.token_ids` from /render). Seeds
|
||||
detokenization from the prompt tail so the first output token keeps its
|
||||
leading space on SentencePiece tokenizers. Falls back to
|
||||
`generate_response.prompt_token_ids`, then to unseeded decoding.
|
||||
|
||||
Only the last few IDs are read, so a suffix of the prompt is enough.
|
||||
"""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
"""The original (post-adjust_request) ChatCompletionRequest from /render.
|
||||
|
||||
Required by the parsing so that tool/reasoning parsers can receive the full
|
||||
request context they expect (request.tools, request.tool_choice,
|
||||
request._grammar_from_parser, etc.).
|
||||
"""
|
||||
# --8<-- [end:derender-chat-request]
|
||||
|
||||
|
||||
class DerenderCompletionRequest(BaseModel):
|
||||
"""Request for the /v1/completions/derender endpoint (non-streaming).
|
||||
|
||||
Parallel to DerenderChatRequest but handles the multi-prompt completions
|
||||
case: one GenerateResponse per prompt, mirroring the list[GenerateRequest]
|
||||
returned by /v1/completions/render.
|
||||
"""
|
||||
|
||||
# --8<-- [start:derender-completion-request]
|
||||
stream: Literal[False] = False
|
||||
|
||||
model: str | None = None
|
||||
"""Served model name. Defaults to the server's served model name."""
|
||||
|
||||
generate_responses: list[GenerateTokensResponse]
|
||||
"""One `output_mode="tokens"` response per prompt, parallel to the
|
||||
list[GenerateRequest] returned by /v1/completions/render."""
|
||||
|
||||
prompt_tokens: list[int] | None = None
|
||||
"""One prompt token count per response; each defaults to 0 if omitted.
|
||||
|
||||
If provided, len(prompt_tokens) must equal len(generate_responses).
|
||||
"""
|
||||
|
||||
prompt_token_ids: list[list[NonNegativeInt] | None] | None = None
|
||||
"""One prompt token ID list per response, used to seed detokenization.
|
||||
See `DerenderChatRequest.prompt_token_ids`.
|
||||
|
||||
If provided, len(prompt_token_ids) must equal len(generate_responses).
|
||||
A `None` entry falls back to `generate_responses[i].prompt_token_ids`.
|
||||
"""
|
||||
|
||||
completion_request: CompletionRequest | None = None
|
||||
"""The original (post-adjust_request) CompletionRequest from /render.
|
||||
|
||||
Mirrors chat_request on DerenderChatRequest. Required by the parsing
|
||||
so parsers receive the full request context.
|
||||
"""
|
||||
# --8<-- [end:derender-completion-request]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_prompt_tokens_length(self) -> "DerenderCompletionRequest":
|
||||
if self.prompt_tokens is not None and len(self.prompt_tokens) != len(
|
||||
self.generate_responses
|
||||
):
|
||||
raise ValueError(
|
||||
f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
|
||||
f"generate_responses length ({len(self.generate_responses)})"
|
||||
)
|
||||
if self.prompt_token_ids is not None and len(self.prompt_token_ids) != len(
|
||||
self.generate_responses
|
||||
):
|
||||
raise ValueError(
|
||||
f"prompt_token_ids length ({len(self.prompt_token_ids)}) must "
|
||||
f"equal generate_responses length ({len(self.generate_responses)})"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class DerenderStreamState(BaseModel):
|
||||
"""Per sequence state for stateless streaming derender.
|
||||
|
||||
The client carries this between successive per chunk HTTP calls to the
|
||||
streaming derender endpoint. All fields are plain JSON serializable data.
|
||||
No opaque tokenizer or parser internals are stored here.
|
||||
|
||||
Two separate sets of fields support two different streaming modes:
|
||||
|
||||
- For plain streaming (no parser configured including the completions path):
|
||||
`prev_tokens`, `prefix_offset` and `read_offset` maintain a bounded incremental
|
||||
decoding window. This requires O(window) transport and O(delta) computation
|
||||
per chunk.
|
||||
- For parser enabled chat streaming: `output_token_ids`, `output_chunk_lens`,
|
||||
`tools_streamed` and `last_tool_call_ids` are used to replay `parse_delta()`
|
||||
from scratch on every chunk because parser state cannot be serialized. This
|
||||
incurs O(n) transport per chunk (O(n²) per generation) and O(n²)
|
||||
`parse_delta()` calls per generation. Since many parsers re-scan the
|
||||
entire accumulated text on each invocation, `parse_delta()` itself is
|
||||
O(n) yielding a true worst case compute cost of O(n³) per generation.
|
||||
No caching is performed. Work is bounded by `max_model_len`.
|
||||
|
||||
The detokenization strategy carries the incremental decode offsets
|
||||
directly rather than re-sending the whole token history each chunk.
|
||||
`detokenize_incrementally` only ever reads the trailing token window
|
||||
`prev_tokens[prefix_offset:]`, so we carry just that tail plus the two
|
||||
offsets. Each chunk resumes exactly where the last one stopped, including
|
||||
any partially processed multi-byte character (tracked by `read_offset`),
|
||||
then trims and rebases the window so it never grows with generation length.
|
||||
|
||||
Performance:
|
||||
- Compute per chunk is O(delta). One `detokenize_incrementally` call per
|
||||
new token, independent of how many tokens preceded it.
|
||||
- Transport per chunk is O(window). The carried tail is bounded by the
|
||||
incremental detokenization offset, so cumulative bytes over the wire are
|
||||
O(n) rather than the O(n^2) a full history round trip would incur.
|
||||
"""
|
||||
|
||||
prev_tokens: list[str] = Field(default_factory=list)
|
||||
"""Trailing decode window. Token strings from `prefix_offset` onward.
|
||||
|
||||
Bounded, trimmed and rebased each chunk to the tail
|
||||
`detokenize_incrementally` still reads, so it does not grow with the
|
||||
number of chunks.
|
||||
"""
|
||||
|
||||
prefix_offset: int = Field(default=0, ge=0)
|
||||
"""Prefix offset into `prev_tokens` for incremental detokenization."""
|
||||
|
||||
read_offset: int = Field(default=0, ge=0)
|
||||
"""Read offset into `prev_tokens` for incremental detokenization."""
|
||||
|
||||
@field_validator("prev_tokens")
|
||||
@classmethod
|
||||
def _bound_prev_tokens(cls, v: list[str]) -> list[str]:
|
||||
# INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET is small (5) and the trimmed
|
||||
# window is O(offset). A generous limit rejects unusually large or malformed
|
||||
# payloads without restricting legitimate multi-byte sequences.
|
||||
limit = 1024
|
||||
if len(v) > limit:
|
||||
raise ValueError(f"prev_tokens length ({len(v)}) exceeds maximum ({limit})")
|
||||
return v
|
||||
|
||||
role_sent: bool = False
|
||||
"""True once the initial `role: "assistant"` delta has been emitted.
|
||||
|
||||
Prevents re-emitting the role on subsequent chunks even when the detok
|
||||
window is transiently empty (e.g. usage only final chunk).
|
||||
"""
|
||||
|
||||
logprob_context_token_ids: list[int] = Field(default_factory=list, max_length=4)
|
||||
"""Trailing sampled token IDs carried across chunks so byte-fallback
|
||||
(U+FFFD) correction during logprob placeholder resolution has context
|
||||
at chunk boundaries. Bounded to the 4-token window that
|
||||
``_correct_decoded_token`` reads."""
|
||||
|
||||
logprob_text_offset: int = Field(default=0, ge=0)
|
||||
"""Cumulative emitted text length. Seeds ``text_offset`` for completion
|
||||
streaming logprobs so offsets stay absolute across chunks, mirroring
|
||||
``initial_text_offset`` in the generate streaming path."""
|
||||
|
||||
output_token_ids: list[int] = Field(default_factory=list)
|
||||
"""All output tokens seen so far. Parser path only.
|
||||
|
||||
Replay buffer: each chunk rebuilds a fresh parser and replays every
|
||||
token in here through `parse_delta` (discarding the result) before
|
||||
processing the current chunk's tokens since parser internal state
|
||||
cannot be serialized into this stateless model. Unavoidably O(n)
|
||||
bounded by `max_model_len` (enforced server side, not by a field
|
||||
validator here since the bound is model dependent).
|
||||
"""
|
||||
|
||||
output_chunk_lens: list[Annotated[int, Field(gt=0)]] = Field(default_factory=list)
|
||||
"""Token count of each chunk in `output_token_ids`. Parser path only.
|
||||
|
||||
Replay uses these to reproduce the original `parse_delta` call
|
||||
boundaries. Must sum to `len(output_token_ids)`.
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_output_chunk_lens(self) -> "DerenderStreamState":
|
||||
total = sum(self.output_chunk_lens)
|
||||
if total != len(self.output_token_ids):
|
||||
raise ValueError(
|
||||
f"output_chunk_lens must sum to len(output_token_ids) "
|
||||
f"(got sum={total}, len(output_token_ids)="
|
||||
f"{len(self.output_token_ids)})"
|
||||
)
|
||||
return self
|
||||
|
||||
tools_streamed: bool = False
|
||||
"""True once a tool call delta has been emitted. Parser path only.
|
||||
|
||||
Drives the `finish_reason` -> `"tool_calls"` rewrite on the final
|
||||
chunk mirroring the generate streaming path.
|
||||
"""
|
||||
|
||||
last_tool_call_ids: list[str] = Field(default_factory=list)
|
||||
"""Stable tool call IDs, assigned once when each call first appears.
|
||||
|
||||
Indexed by tool call index. Parser path only. Prevents ID regeneration
|
||||
when replay reprocesses a tool call that already has a pinned ID.
|
||||
"""
|
||||
|
||||
|
||||
class DerenderChatStreamRequest(BaseModel):
|
||||
"""One chunk streaming derender request for /v1/chat/completions/derender.
|
||||
|
||||
The client sends one request per SSE chunk received from
|
||||
`/inference/v1/generate`. Each request carries the generate chunk
|
||||
plus the `stream_state` returned by the previous call (`None` on the
|
||||
first call). The response contains the derendered chunk and the updated
|
||||
state to be passed to the next call.
|
||||
|
||||
This implements stateless no server side session. All mutable state lives in
|
||||
the client carried `stream_state`.
|
||||
"""
|
||||
|
||||
# --8<-- [start:derender-chat-stream-request]
|
||||
stream: Literal[True]
|
||||
|
||||
model: str | None = None
|
||||
generate_chunk: GenerateTokensStreamResponse
|
||||
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`
|
||||
(`stream=True`)."""
|
||||
|
||||
stream_state: DerenderStreamState | None = None
|
||||
"""Client carried detok state from the previous call. `None` on first."""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage. Forwarded from the render step."""
|
||||
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
"""Prompt token IDs. Required by the parser path's `parse_delta` to
|
||||
settle its initial reasoning state (e.g. chat templates that pre-open
|
||||
`<think>`). `prompt_tokens` is a usage count and cannot serve this
|
||||
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
|
||||
|
||||
Rejected with a 400 when a tool or reasoning
|
||||
parser is configured and this is omitted. Without it, `parse_delta`
|
||||
cannot tell whether the prompt left reasoning open and would silently
|
||||
misclassify reasoning content as plain content. On all paths it also
|
||||
seeds detokenization on the first chunk, falling back to
|
||||
`generate_chunk.prompt_token_ids` when omitted (see
|
||||
`DerenderChatRequest.prompt_token_ids`).
|
||||
|
||||
With a parser configured, send the full list. Parsers look back to the
|
||||
last reasoning marker which can be anywhere in the prompt, so a suffix
|
||||
can flip the initial reasoning state. Without a parser, a suffix is
|
||||
enough.
|
||||
"""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
"""The original (post adjust_request) ChatCompletionRequest from /render."""
|
||||
# --8<-- [end:derender-chat-stream-request]
|
||||
|
||||
|
||||
class DerenderCompletionStreamRequest(BaseModel):
|
||||
"""One chunk streaming derender request for /v1/completions/derender.
|
||||
|
||||
Parallel to `DerenderChatStreamRequest` for the completions endpoint.
|
||||
Each call processes one SSE chunk (one output sequence's delta) and
|
||||
returns the derendered chunk plus updated state.
|
||||
"""
|
||||
|
||||
# --8<-- [start:derender-completion-stream-request]
|
||||
stream: Literal[True]
|
||||
|
||||
model: str | None = None
|
||||
generate_chunk: GenerateTokensStreamResponse
|
||||
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`."""
|
||||
|
||||
stream_state: DerenderStreamState | None = None
|
||||
"""Client-carried detok state. `None` on the first call."""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage."""
|
||||
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
"""Prompt token IDs, used on the first chunk to seed detokenization.
|
||||
Falls back to `generate_chunk.prompt_token_ids`. See
|
||||
`DerenderChatRequest.prompt_token_ids`.
|
||||
"""
|
||||
|
||||
completion_request: CompletionRequest | None = None
|
||||
"""The original (post adjust_request) CompletionRequest from /render."""
|
||||
# --8<-- [end:derender-completion-stream-request]
|
||||
|
||||
|
||||
class DerenderChatStreamResponse(BaseModel):
|
||||
"""Response for one streaming chat derender chunk.
|
||||
|
||||
Pairs the derendered SSE chunk with the updated client carried state to
|
||||
pass to the next call.
|
||||
"""
|
||||
|
||||
chunk: ChatCompletionStreamResponse
|
||||
stream_state: DerenderStreamState
|
||||
|
||||
|
||||
class DerenderCompletionStreamResponse(BaseModel):
|
||||
"""Response for one streaming completions derender chunk.
|
||||
|
||||
Parallel to `DerenderChatStreamResponse` for the completions endpoint.
|
||||
"""
|
||||
|
||||
chunk: CompletionStreamResponse
|
||||
stream_state: DerenderStreamState
|
||||
|
||||
|
||||
# Determines the type by checking the `stream` field's literal value. A body without
|
||||
# `stream` validates as the non-streaming member
|
||||
# (`stream` defaults to `False` there), so FastAPI can validate and dispatch both
|
||||
# shapes on a single path.
|
||||
DerenderChatRequestUnion: TypeAlias = DerenderChatRequest | DerenderChatStreamRequest
|
||||
DerenderCompletionRequestUnion: TypeAlias = (
|
||||
DerenderCompletionRequest | DerenderCompletionStreamRequest
|
||||
)
|
||||
@@ -1,810 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProb,
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionLogProbsContent,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
PromptTokenUsageInfo,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput, TokensPrompt, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.chat_utils import AsyncMultiModalItemTracker
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.serial_utils import numpy2base64
|
||||
|
||||
from .mm_features import (
|
||||
mm_kwargs_from_features,
|
||||
placeholder_ranges_from_engine_input,
|
||||
)
|
||||
from .protocol import (
|
||||
GenerateLogProb,
|
||||
GenerateLogProbs,
|
||||
GenerateLogProbsContent,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateTextChoice,
|
||||
GenerateTextResponse,
|
||||
GenerateTextStreamChoice,
|
||||
GenerateTextStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamChoice,
|
||||
GenerateTokensStreamResponse,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _clamp_logprob(logprob: float) -> float:
|
||||
"""The OpenAI shapes' floor for a logprob: ``-inf`` and NaN become ``-9999.0``.
|
||||
|
||||
``max(nan, -9999.0)`` is NaN in Python, so NaN needs its own check; NaN
|
||||
would otherwise fail ``JSONResponse`` or reach the client as ``null``.
|
||||
"""
|
||||
return -9999.0 if math.isnan(logprob) else max(logprob, -9999.0)
|
||||
|
||||
|
||||
def _logprob_token(
|
||||
token_id: int, logprob: Logprob | None, tokenizer: TokenizerLike | None
|
||||
) -> tuple[str, list[int] | None]:
|
||||
"""Token string and UTF-8 bytes for one logprob entry.
|
||||
|
||||
Without a tokenizer the token is a ``token_id:N`` placeholder with no bytes.
|
||||
"""
|
||||
if tokenizer is None:
|
||||
return format_token_id_placeholder(token_id), None
|
||||
token = (
|
||||
tokenizer.decode([token_id])
|
||||
if logprob is None
|
||||
else GenerateBaseServing._get_decoded_token(logprob, token_id, tokenizer)
|
||||
)
|
||||
return token, list(token.encode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def _top_logprob(
|
||||
token_id: int, logprob: Logprob, tokenizer: TokenizerLike | None
|
||||
) -> ChatCompletionLogProb:
|
||||
token, token_bytes = _logprob_token(token_id, logprob, tokenizer)
|
||||
return ChatCompletionLogProb(
|
||||
token=token, logprob=_clamp_logprob(logprob.logprob), bytes=token_bytes
|
||||
)
|
||||
|
||||
|
||||
class ServingTokens(GenerateBaseServing):
|
||||
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
online_renderer: OnlineRenderer,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
force_no_detokenize: bool = False,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
self.online_renderer = online_renderer
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_log_outputs = enable_log_outputs
|
||||
self.force_no_detokenize = force_no_detokenize
|
||||
self.has_tokenizer = (
|
||||
not force_no_detokenize and self.renderer.tokenizer is not None
|
||||
)
|
||||
if force_no_detokenize:
|
||||
logger.info(
|
||||
"Tokens-only mode is enabled, skipping detokenization "
|
||||
"step for incoming requests."
|
||||
)
|
||||
|
||||
# Mirrors ``OpenAIServingChat`` so we can apply server-side
|
||||
# ``max_tokens`` defaulting when the client omits it. Without this,
|
||||
# ``SamplingParams.max_tokens`` falls back to its dataclass default
|
||||
# of 16 and silently truncates every generation.
|
||||
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
||||
mc = self.model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
def _validate_mm_cache_handles(
|
||||
self,
|
||||
mm_kwargs: dict[str, list[MultiModalKwargsItem | None]],
|
||||
mm_hashes: dict[str, list[str]],
|
||||
) -> ErrorResponse | None:
|
||||
cache = self.online_renderer.renderer.mm_processor_cache
|
||||
if cache is None:
|
||||
return None
|
||||
try:
|
||||
for modality, items in mm_kwargs.items():
|
||||
for mm_hash, item in zip(mm_hashes[modality], items, strict=True):
|
||||
if item is not None:
|
||||
cache.validate_input_item(item, mm_hash)
|
||||
except ValueError as error:
|
||||
return self.create_error_response(error)
|
||||
return None
|
||||
|
||||
async def serve_tokens(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> GenerateResponse | ErrorResponse | AsyncGenerator[str, None]:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
self._preflight()
|
||||
|
||||
lora_request = None
|
||||
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
|
||||
|
||||
model_name = self.models.model_name(lora_request)
|
||||
|
||||
request_id = (
|
||||
f"generate-tokens-{self._base_request_id(raw_request, request.request_id)}"
|
||||
)
|
||||
|
||||
request_metadata = RequestResponseMetadata(request_id=request_id)
|
||||
if raw_request:
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
sampling_params = request.sampling_params
|
||||
max_num_seqs = self.engine_client.vllm_config.scheduler_config.max_num_seqs
|
||||
if sampling_params.n > max_num_seqs:
|
||||
return self.create_error_response(
|
||||
f"sampling_params.n must be at most the server's max_num_seqs "
|
||||
f"({max_num_seqs}), got {sampling_params.n}."
|
||||
)
|
||||
# The stream schema has no field for the scores.
|
||||
if request.stream and sampling_params.prompt_logprob_token_ids is not None:
|
||||
return self.create_error_response(
|
||||
"prompt_logprob_token_ids are not available when stream=true."
|
||||
)
|
||||
if self.force_no_detokenize and sampling_params.stop:
|
||||
# SamplingParams rejects stop with detokenize=False at request
|
||||
# validation, but this server forces detokenize=False afterwards,
|
||||
# so the combination must be rejected here or stop strings are
|
||||
# silently never applied.
|
||||
return self.create_error_response(
|
||||
"stop strings are not supported on a --tokens-only server "
|
||||
"because detokenization is disabled. Check stop strings on "
|
||||
"the coordinator, or use stop_token_ids."
|
||||
)
|
||||
if request.output_mode != "tokens" and not self.has_tokenizer:
|
||||
# The no-op detokenizer returns "", so without this check the
|
||||
# request would succeed with empty text.
|
||||
return self.create_error_response(
|
||||
f"output_mode={request.output_mode!r} requires a tokenizer, but "
|
||||
"this server does not load one (--tokens-only or "
|
||||
"--skip-tokenizer-init). Send the request to a server that "
|
||||
"loads a tokenizer, or use output_mode='tokens'."
|
||||
)
|
||||
try:
|
||||
msgspec.msgpack.encode(
|
||||
(
|
||||
sampling_params,
|
||||
request.kv_transfer_params,
|
||||
request.ec_transfer_params,
|
||||
)
|
||||
)
|
||||
except (OverflowError, TypeError, ValueError) as e:
|
||||
return self.create_error_response(e)
|
||||
|
||||
engine_input: EngineInput
|
||||
if request.content_parts:
|
||||
tracker = AsyncMultiModalItemTracker(self.model_config)
|
||||
mm_parser = tracker.create_parser()
|
||||
for part in request.content_parts:
|
||||
ptype = part.get("type", "")
|
||||
url = part.get("url")
|
||||
uuid = part.get("uuid")
|
||||
if ptype == "image_url":
|
||||
mm_parser.parse_image(url, uuid)
|
||||
elif ptype == "audio_url":
|
||||
mm_parser.parse_audio(url, uuid)
|
||||
elif ptype == "video_url":
|
||||
mm_parser.parse_video(url, uuid)
|
||||
mm_data, mm_uuids = await tracker.resolve_items()
|
||||
prompt = TokensPrompt(prompt_token_ids=request.token_ids)
|
||||
if request.cache_salt is not None:
|
||||
prompt["cache_salt"] = request.cache_salt
|
||||
if mm_data:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
(engine_input,) = await self.online_renderer.renderer.render_cmpl_async(
|
||||
[prompt]
|
||||
)
|
||||
elif features := request.features:
|
||||
# Convert PlaceholderRangeInfo → PlaceholderRange per modality.
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {
|
||||
modality: [
|
||||
PlaceholderRange(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in features.mm_placeholders.items()
|
||||
}
|
||||
|
||||
# Deserialize full tensor data and optional metadata-only data.
|
||||
# Metadata-only items are valid when ec_transfer_params is set.
|
||||
mm_kwargs = mm_kwargs_from_features(features)
|
||||
if error := self._validate_mm_cache_handles(mm_kwargs, features.mm_hashes):
|
||||
return error
|
||||
|
||||
engine_input = mm_input(
|
||||
prompt_token_ids=request.token_ids,
|
||||
mm_kwargs=MultiModalKwargsItems(mm_kwargs),
|
||||
mm_hashes=features.mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
cache_salt=request.cache_salt,
|
||||
)
|
||||
else:
|
||||
(engine_input,) = await self.online_renderer.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.token_ids,
|
||||
prompt_embeds=None,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
|
||||
# Offsets are relative to the decoder prompt, so they are not
|
||||
# meaningful for encoder-decoder models.
|
||||
request._response_mm_placeholders = (
|
||||
placeholder_ranges_from_engine_input(engine_input)
|
||||
if request.return_token_ids and not self.model_config.is_encoder_decoder
|
||||
else None
|
||||
)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
||||
|
||||
# Pass disaggregated-serving parameters through to the engine.
|
||||
if request.kv_transfer_params is not None:
|
||||
extra = sampling_params.extra_args or {}
|
||||
extra["kv_transfer_params"] = request.kv_transfer_params
|
||||
sampling_params.extra_args = extra
|
||||
if request.ec_transfer_params is not None:
|
||||
extra = sampling_params.extra_args or {}
|
||||
extra["ec_transfer_params"] = request.ec_transfer_params
|
||||
sampling_params.extra_args = extra
|
||||
|
||||
# Apply server-side ``max_tokens`` defaulting when the client did
|
||||
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
|
||||
# defaults ``max_tokens`` to 16, which would otherwise silently cap
|
||||
# every generation that omits the field.
|
||||
if not request.is_sampling_param_provided("max_tokens"):
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
max_model_len=self.model_config.max_model_len,
|
||||
max_tokens=None,
|
||||
input_length=self._extract_prompt_len(engine_input),
|
||||
default_sampling_params=self.default_sampling_params,
|
||||
override_max_tokens=self.override_max_tokens,
|
||||
)
|
||||
|
||||
if self.force_no_detokenize:
|
||||
sampling_params.detokenize = False
|
||||
sampling_params.output_kind = (
|
||||
RequestOutputKind.DELTA if request.stream else RequestOutputKind.FINAL_ONLY
|
||||
)
|
||||
|
||||
self._log_inputs(
|
||||
request_id,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
# Extract data_parallel_rank from header (router can inject it)
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
session_id = self._get_session_id_from_headers(raw_request)
|
||||
|
||||
result_generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
reasoning_ended=request.reasoning_ended,
|
||||
reasoning_parser_kwargs=(
|
||||
request.reasoning_parser_kwargs.model_dump()
|
||||
if request.reasoning_parser_kwargs is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
assert result_generator is not None
|
||||
|
||||
if request.stream:
|
||||
return self.serve_tokens_stream_generator(
|
||||
request,
|
||||
result_generator,
|
||||
request_id,
|
||||
model_name,
|
||||
request_metadata,
|
||||
)
|
||||
|
||||
return await self.serve_tokens_full_generator(
|
||||
request, result_generator, request_id, model_name, request_metadata
|
||||
)
|
||||
|
||||
async def serve_tokens_full_generator(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
result_generator: AsyncGenerator[RequestOutput, None],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
) -> ErrorResponse | GenerateResponse:
|
||||
created_time = int(time.time())
|
||||
final_res: RequestOutput | None = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
text_mode = request.output_mode == "text"
|
||||
tokenizer = self._logprobs_tokenizer(text_mode)
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
final_res = res
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
|
||||
assert final_res is not None
|
||||
|
||||
tokens_choices: list[GenerateTokensChoice] = []
|
||||
text_choices: list[GenerateTextChoice] = []
|
||||
num_generated_tokens = 0
|
||||
for output in final_res.outputs:
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
# This is top_logprobs in completions API
|
||||
if sampling_params.logprobs is not None:
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
|
||||
if text_mode:
|
||||
logprobs = self._create_text_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
)
|
||||
|
||||
sampling_mask = None
|
||||
if output.sampling_mask is not None:
|
||||
sampling_mask = output.sampling_mask.token_ids
|
||||
|
||||
choice_fields: dict[str, Any] = dict(
|
||||
index=output.index,
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason if output.finish_reason else "stop",
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=as_list(output.token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
weight_versions=output.weight_versions,
|
||||
)
|
||||
if text_mode:
|
||||
text_choices.append(
|
||||
GenerateTextChoice(text=output.text, **choice_fields)
|
||||
)
|
||||
else:
|
||||
tokens_choices.append(GenerateTokensChoice(**choice_fields))
|
||||
num_generated_tokens += len(output.token_ids)
|
||||
|
||||
assert final_res.prompt_token_ids is not None
|
||||
num_prompt_tokens = len(final_res.prompt_token_ids)
|
||||
if final_res.encoder_prompt_token_ids is not None:
|
||||
num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
)
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and final_res.num_cached_tokens is not None
|
||||
):
|
||||
# This info is not available at the /coordinator level
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=final_res.num_cached_tokens
|
||||
)
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
per_request_metrics = None
|
||||
if request.sampling_params.n == 1:
|
||||
spec_stats = build_spec_decoding_metrics(final_res)
|
||||
if spec_stats is not None:
|
||||
per_request_metrics = PerRequestMetrics(speculative_decoding=spec_stats)
|
||||
response_fields: dict[str, Any] = dict(
|
||||
request_id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
usage=usage,
|
||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||
prompt_token_id_logprobs=(
|
||||
numpy2base64(final_res.prompt_token_id_logprobs)
|
||||
if final_res.prompt_token_id_logprobs is not None
|
||||
else None
|
||||
),
|
||||
prompt_token_ids=(
|
||||
final_res.prompt_token_ids if request.return_token_ids else None
|
||||
),
|
||||
mm_placeholders=request._response_mm_placeholders,
|
||||
metrics=per_request_metrics,
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
ec_transfer_params=final_res.ec_transfer_params,
|
||||
)
|
||||
response: GenerateTokensResponse | GenerateTextResponse
|
||||
if text_mode:
|
||||
response = GenerateTextResponse(choices=text_choices, **response_fields)
|
||||
else:
|
||||
response = GenerateTokensResponse(choices=tokens_choices, **response_fields)
|
||||
|
||||
# Log complete response if output logging is enabled
|
||||
if self.enable_log_outputs and self.request_logger:
|
||||
for choice in response.choices:
|
||||
# Get the corresponding output token IDs
|
||||
output_token_ids = None
|
||||
if choice.index < len(final_res.outputs):
|
||||
output_token_ids = final_res.outputs[choice.index].token_ids
|
||||
|
||||
if output_token_ids:
|
||||
# Log token_ids only.
|
||||
self.request_logger.log_outputs(
|
||||
request_id=request_id,
|
||||
outputs="",
|
||||
output_token_ids=output_token_ids,
|
||||
finish_reason=choice.finish_reason,
|
||||
is_streaming=False,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
async def serve_tokens_stream_generator(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
result_generator: AsyncGenerator[RequestOutput, None],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
num_prompt_tokens = 0
|
||||
first_iteration = True
|
||||
prompt_token_ids: list[int] | None = None
|
||||
num_cached_tokens = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
# With n > 1, the first output may not include every choice yet.
|
||||
num_generated_tokens = [0] * sampling_params.n
|
||||
last_res: RequestOutput | None = None
|
||||
text_mode = request.output_mode == "text"
|
||||
tokenizer = self._logprobs_tokenizer(text_mode)
|
||||
|
||||
include_usage, include_continuous_usage = should_include_usage(
|
||||
request.stream_options, False
|
||||
)
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
last_res = res
|
||||
if first_iteration:
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
if request.return_token_ids:
|
||||
prompt_token_ids = res.prompt_token_ids
|
||||
if res.encoder_prompt_token_ids is not None:
|
||||
num_prompt_tokens += len(res.encoder_prompt_token_ids)
|
||||
num_cached_tokens = res.num_cached_tokens
|
||||
first_iteration = False
|
||||
|
||||
for output in res.outputs:
|
||||
i = output.index
|
||||
delta_token_ids = output.token_ids
|
||||
num_generated_tokens[i] += len(delta_token_ids)
|
||||
|
||||
finish_reason = output.finish_reason
|
||||
self._raise_if_error(finish_reason, request_id)
|
||||
|
||||
# Terminal outputs are always emitted so the client sees
|
||||
# the finish reason, e.g. an abort, which has no new
|
||||
# tokens. Text mode also emits text held back for stop
|
||||
# string matching, which can arrive without token IDs.
|
||||
if not (
|
||||
delta_token_ids
|
||||
or finish_reason is not None
|
||||
or (text_mode and output.text)
|
||||
):
|
||||
continue
|
||||
|
||||
if sampling_params.logprobs is not None:
|
||||
out_logprobs = output.logprobs
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
|
||||
if text_mode:
|
||||
logprobs = self._create_text_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
routed_experts_b64 = (
|
||||
numpy2base64(output.routed_experts)
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
)
|
||||
|
||||
sampling_mask = None
|
||||
if output.sampling_mask is not None:
|
||||
sampling_mask = output.sampling_mask.token_ids
|
||||
choice_fields: dict[str, Any] = dict(
|
||||
index=i,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=as_list(delta_token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
weight_versions=output.weight_versions,
|
||||
)
|
||||
chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
|
||||
if text_mode:
|
||||
chunk = GenerateTextStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
GenerateTextStreamChoice(
|
||||
text=output.text, **choice_fields
|
||||
)
|
||||
],
|
||||
)
|
||||
else:
|
||||
chunk = GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[GenerateTokensStreamChoice(**choice_fields)],
|
||||
)
|
||||
|
||||
if prompt_token_ids is not None:
|
||||
chunk.prompt_token_ids = prompt_token_ids
|
||||
chunk.mm_placeholders = request._response_mm_placeholders
|
||||
prompt_token_ids = None
|
||||
if include_continuous_usage:
|
||||
chunk.usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=num_generated_tokens[i],
|
||||
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
|
||||
)
|
||||
|
||||
# Omit fields that are absent from token-bearing chunks.
|
||||
exclude = {
|
||||
name
|
||||
for name in ("prompt_token_ids", "mm_placeholders", "metrics")
|
||||
if getattr(chunk, name) is None
|
||||
}
|
||||
yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n"
|
||||
|
||||
total_completion_tokens = sum(num_generated_tokens)
|
||||
final_usage_info = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=num_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
per_request_metrics = None
|
||||
if sampling_params.n == 1:
|
||||
spec_stats = build_spec_decoding_metrics(last_res)
|
||||
if spec_stats is not None:
|
||||
per_request_metrics = PerRequestMetrics(
|
||||
speculative_decoding=spec_stats
|
||||
)
|
||||
final_chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
|
||||
if text_mode:
|
||||
final_chunk = GenerateTextStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
else:
|
||||
final_chunk = GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
request_metadata.final_usage_info = final_usage_info
|
||||
|
||||
except GenerationError as e:
|
||||
yield (
|
||||
f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error in token generation stream.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
def _logprobs_tokenizer(self, text_mode: bool) -> TokenizerLike | None:
|
||||
"""Tokenizer that resolves logprob tokens, or None for placeholders.
|
||||
|
||||
``--return-tokens-as-token-ids`` keeps placeholders at every level, as
|
||||
it does on ``/v1/completions``.
|
||||
"""
|
||||
if not text_mode or self.return_tokens_as_token_ids:
|
||||
return None
|
||||
return self.renderer.tokenizer
|
||||
|
||||
def _create_text_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Create OpenAI-style logprobs for ``output_mode="text"``.
|
||||
|
||||
With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``;
|
||||
without one (``--return-tokens-as-token-ids``) they are ``token_id:N``
|
||||
placeholders. Like tokens mode, ``top_logprobs`` carries every candidate
|
||||
the engine returned (the sampled token first, then the top k), not cut
|
||||
to the requested count.
|
||||
"""
|
||||
logprobs_content: list[ChatCompletionLogProbsContent] = []
|
||||
|
||||
for i, token_id in enumerate(token_ids):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
token, token_bytes = _logprob_token(token_id, None, tokenizer)
|
||||
logprobs_content.append(
|
||||
ChatCompletionLogProbsContent(token=token, bytes=token_bytes)
|
||||
)
|
||||
else:
|
||||
step_token = step_top_logprobs[token_id]
|
||||
token, token_bytes = _logprob_token(token_id, step_token, tokenizer)
|
||||
|
||||
logprobs_content.append(
|
||||
ChatCompletionLogProbsContent(
|
||||
token=token,
|
||||
logprob=_clamp_logprob(step_token.logprob),
|
||||
bytes=token_bytes,
|
||||
top_logprobs=[
|
||||
_top_logprob(top_id, logprob, tokenizer)
|
||||
for top_id, logprob in step_top_logprobs.items()
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
return ChatCompletionLogProbs(content=logprobs_content)
|
||||
|
||||
def _create_tokens_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
) -> GenerateLogProbs:
|
||||
"""Create generate-shaped logprobs (integer token ids, no tokenizer).
|
||||
|
||||
``top_logprobs`` carries every candidate the engine returned for the
|
||||
position: the sampled token first, then the top k, so k entries when the
|
||||
sampled token is in the top k and k + 1 when it is not. It is not cut to
|
||||
the requested count: `/v1/chat/completions` and
|
||||
`/v1/completions` cut differently, and derender applies each endpoint's
|
||||
cut the way the coupled server does.
|
||||
|
||||
The engine reports rank 0 for a sampled token whose logprob is NaN; that
|
||||
is not a rank, so it is sent as ``None`` (the logprob is clamped).
|
||||
"""
|
||||
logprobs_content: list[GenerateLogProbsContent] = []
|
||||
|
||||
for i, token_id in enumerate(token_ids):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
# Same sentinel the OpenAI shapes use when the sampled token
|
||||
# has no entry in the engine's top-k map.
|
||||
logprobs_content.append(
|
||||
GenerateLogProbsContent(token_id=token_id, logprob=-9999.0)
|
||||
)
|
||||
else:
|
||||
step_token = step_top_logprobs[token_id]
|
||||
|
||||
logprobs_content.append(
|
||||
GenerateLogProbsContent(
|
||||
token_id=token_id,
|
||||
logprob=_clamp_logprob(step_token.logprob),
|
||||
rank=step_token.rank or None,
|
||||
top_logprobs=[
|
||||
GenerateLogProb(
|
||||
token_id=top_token_id,
|
||||
logprob=_clamp_logprob(top_logprob.logprob),
|
||||
rank=top_logprob.rank or None,
|
||||
)
|
||||
for top_token_id, top_logprob in step_top_logprobs.items()
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
return GenerateLogProbs(content=logprobs_content)
|
||||
@@ -16,12 +16,6 @@ def register_vllm_serve_api_routers(app: FastAPI):
|
||||
|
||||
register_instrumentator_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.serve.lora.api_router import (
|
||||
attach_router as attach_lora_router,
|
||||
)
|
||||
|
||||
attach_lora_router(app)
|
||||
|
||||
from vllm.entrypoints.serve.profile.api_router import (
|
||||
attach_router as attach_profile_router,
|
||||
)
|
||||
|
||||
@@ -15,9 +15,7 @@ from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMNotFoundError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
@@ -52,27 +50,11 @@ class BaseServing:
|
||||
if self.request_logger is not None:
|
||||
self.request_logger.log_request_body(request)
|
||||
|
||||
error_response = None
|
||||
|
||||
if self._is_model_supported(request.model):
|
||||
return None
|
||||
if request.model in self.models.lora_requests:
|
||||
return None
|
||||
if (
|
||||
envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
|
||||
and request.model
|
||||
and (load_result := await self.models.resolve_lora(request.model))
|
||||
):
|
||||
if isinstance(load_result, LoRARequest):
|
||||
return None
|
||||
if (
|
||||
isinstance(load_result, ErrorResponse)
|
||||
and load_result.error.code == HTTPStatus.BAD_REQUEST.value
|
||||
):
|
||||
error_response = load_result
|
||||
|
||||
served_names = ", ".join(model.name for model in self.models.base_model_paths)
|
||||
return error_response or self.create_error_response(
|
||||
return self.create_error_response(
|
||||
message=(
|
||||
f"The model `{request.model}` does not exist. "
|
||||
f"Valid aliases: {served_names}."
|
||||
@@ -112,7 +94,6 @@ class BaseServing:
|
||||
request_id: str,
|
||||
inputs: PromptType | EngineInput,
|
||||
params: SamplingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
if self.request_logger is None:
|
||||
return
|
||||
@@ -125,7 +106,6 @@ class BaseServing:
|
||||
components.token_ids,
|
||||
components.embeds,
|
||||
params=params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -139,72 +119,3 @@ class BaseServing:
|
||||
return req_id
|
||||
|
||||
return random_uuid() if default is None else default
|
||||
|
||||
def _get_message_types(self, request: AnyRequest) -> set[str]:
|
||||
"""Retrieve the set of types from message content dicts up
|
||||
until `_`; we use this to match potential multimodal data
|
||||
with default per modality loras.
|
||||
"""
|
||||
message_types: set[str] = set()
|
||||
|
||||
if not hasattr(request, "messages"):
|
||||
return message_types
|
||||
|
||||
messages = request.messages
|
||||
if messages is None or isinstance(messages, (str, bytes)):
|
||||
return message_types
|
||||
|
||||
for message in messages:
|
||||
if (
|
||||
isinstance(message, dict)
|
||||
and "content" in message
|
||||
and isinstance(message["content"], list)
|
||||
):
|
||||
for content_dict in message["content"]:
|
||||
if "type" in content_dict:
|
||||
message_types.add(content_dict["type"].split("_")[0])
|
||||
return message_types
|
||||
|
||||
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
|
||||
"""Determine if there are any active default multimodal loras."""
|
||||
# TODO: Currently this is only enabled for chat completions
|
||||
# to be better aligned with only being enabled for .generate
|
||||
# when run offline. It would be nice to support additional
|
||||
# tasks types in the future.
|
||||
message_types = self._get_message_types(request)
|
||||
default_mm_loras = set()
|
||||
|
||||
for lora in self.models.lora_requests.values():
|
||||
# Best effort match for default multimodal lora adapters;
|
||||
# There is probably a better way to do this, but currently
|
||||
# this matches against the set of 'types' in any content lists
|
||||
# up until '_', e.g., to match audio_url -> audio
|
||||
if lora.lora_name in message_types:
|
||||
default_mm_loras.add(lora)
|
||||
|
||||
# Currently only support default modality specific loras if
|
||||
# we have exactly one lora matched on the request.
|
||||
if len(default_mm_loras) == 1:
|
||||
return default_mm_loras.pop()
|
||||
return None
|
||||
|
||||
def _maybe_get_adapters(
|
||||
self,
|
||||
request: AnyRequest,
|
||||
supports_default_mm_loras: bool = False,
|
||||
) -> LoRARequest | None:
|
||||
if request.model in self.models.lora_requests:
|
||||
return self.models.lora_requests[request.model]
|
||||
|
||||
# Currently only support default modality specific loras
|
||||
# if we have exactly one lora matched on the request.
|
||||
if supports_default_mm_loras:
|
||||
default_mm_lora = self._get_active_default_mm_loras(request)
|
||||
if default_mm_lora is not None:
|
||||
return default_mm_lora
|
||||
|
||||
if self._is_model_supported(request.model):
|
||||
return None
|
||||
|
||||
# if _check_model has been called earlier, this will be unreachable
|
||||
raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
|
||||
|
||||
@@ -14,25 +14,12 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
DetokenizeRequest,
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
TokenizeResponse,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
|
||||
from vllm.renderers import ChatParams, TokenizeParams
|
||||
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
|
||||
|
||||
@@ -55,32 +42,16 @@ CompletionLikeRequest: TypeAlias = (
|
||||
CompletionRequest
|
||||
| TokenizeCompletionRequest
|
||||
| DetokenizeRequest
|
||||
| DerenderCompletionRequest
|
||||
| DerenderCompletionStreamRequest
|
||||
)
|
||||
|
||||
ChatLikeRequest: TypeAlias = (
|
||||
ChatCompletionRequest
|
||||
| BatchChatCompletionRequest
|
||||
| TokenizeChatRequest
|
||||
| DerenderChatRequest
|
||||
| DerenderChatStreamRequest
|
||||
)
|
||||
|
||||
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
|
||||
|
||||
AnyRequest: TypeAlias = (
|
||||
CompletionLikeRequest
|
||||
| ChatLikeRequest
|
||||
| SpeechToTextRequest
|
||||
| ResponsesRequest
|
||||
| GenerateRequest
|
||||
)
|
||||
AnyRequest: TypeAlias = CompletionLikeRequest | ChatLikeRequest | ResponsesRequest
|
||||
|
||||
AnyResponse: TypeAlias = (
|
||||
CompletionResponse
|
||||
| ChatCompletionResponse
|
||||
| TranscriptionResponse
|
||||
| TokenizeResponse
|
||||
| GenerateResponse
|
||||
CompletionResponse | ChatCompletionResponse | TokenizeResponse
|
||||
)
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.models.api_router import models
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
UnloadLoRAAdapterRequest,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
"""Attach the LoRA adapter load/unload endpoints to the API server.
|
||||
|
||||
Does nothing when dynamic LoRA updating is disabled. Handler levels are
|
||||
snapshotted and restored because importing
|
||||
model_hosting_container_standards may reconfigure root logging.
|
||||
"""
|
||||
if not envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
|
||||
"""If LoRA dynamic loading & unloading is not enabled, do nothing."""
|
||||
return
|
||||
logger.warning(
|
||||
"LoRA dynamic loading & unloading is enabled in the API server. "
|
||||
"This should ONLY be used for local development!"
|
||||
)
|
||||
|
||||
snapshot = [
|
||||
(h, h.level)
|
||||
for lg in (logging.getLogger(), logging.getLogger("vllm"))
|
||||
for h in lg.handlers
|
||||
]
|
||||
|
||||
try:
|
||||
_attach_router(app)
|
||||
finally:
|
||||
for handler, level in snapshot:
|
||||
handler.setLevel(level)
|
||||
|
||||
|
||||
def _attach_router(app: FastAPI):
|
||||
"""Register the LoRA adapter load/unload routes on the app."""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
@sagemaker_standards.register_load_adapter_handler(
|
||||
request_shape={
|
||||
"lora_name": "body.name",
|
||||
"lora_path": "body.src",
|
||||
"load_inplace": "body.load_inplace || `false`",
|
||||
"is_3d_lora_weight": "body.is_3d_lora_weight || `false`",
|
||||
},
|
||||
)
|
||||
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
|
||||
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
|
||||
"""Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
|
||||
serving engine."""
|
||||
handler: OpenAIServingModels = models(raw_request)
|
||||
response = await handler.load_lora_adapter(request)
|
||||
if isinstance(response, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=response.model_dump(), status_code=response.error.code
|
||||
)
|
||||
|
||||
return Response(status_code=200, content=response)
|
||||
|
||||
@sagemaker_standards.register_unload_adapter_handler(
|
||||
request_shape={
|
||||
"lora_name": "path_params.adapter_name",
|
||||
}
|
||||
)
|
||||
@router.post(
|
||||
"/v1/unload_lora_adapter", dependencies=[Depends(validate_json_request)]
|
||||
)
|
||||
async def unload_lora_adapter(
|
||||
request: UnloadLoRAAdapterRequest, raw_request: Request
|
||||
):
|
||||
"""Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
|
||||
the serving engine."""
|
||||
handler: OpenAIServingModels = models(raw_request)
|
||||
response = await handler.unload_lora_adapter(request)
|
||||
if isinstance(response, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=response.model_dump(), status_code=response.error.code
|
||||
)
|
||||
|
||||
return Response(status_code=200, content=response)
|
||||
|
||||
# register the router
|
||||
app.include_router(router)
|
||||
@@ -1,16 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class LoadLoRAAdapterRequest(BaseModel):
|
||||
lora_name: str
|
||||
lora_path: str
|
||||
load_inplace: bool = False
|
||||
is_3d_lora_weight: bool = False
|
||||
|
||||
|
||||
class UnloadLoRAAdapterRequest(BaseModel):
|
||||
lora_name: str
|
||||
lora_int_id: int | None = Field(default=None)
|
||||
@@ -65,8 +65,6 @@ class ServingTokenization(BaseServing):
|
||||
|
||||
request_id = f"tokenize-{self._base_request_id(raw_request)}"
|
||||
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
if isinstance(request, TokenizeChatRequest):
|
||||
tool_dicts = (
|
||||
None
|
||||
@@ -104,7 +102,6 @@ class ServingTokenization(BaseServing):
|
||||
request_id,
|
||||
engine_input,
|
||||
params=None,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
prompt_components = self._extract_prompt_components(engine_input)
|
||||
@@ -134,13 +131,10 @@ class ServingTokenization(BaseServing):
|
||||
|
||||
request_id = f"tokenize-{self._base_request_id(raw_request)}"
|
||||
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
self._log_inputs(
|
||||
request_id,
|
||||
tokens_input(request.tokens),
|
||||
params=None,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
tok_prompt = await self.renderer.tokenize_prompt_async(
|
||||
|
||||
@@ -20,7 +20,6 @@ from starlette.background import BackgroundTask, BackgroundTasks
|
||||
from vllm import envs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.generate.base.protocol import StreamOptions
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.logger import current_formatter_type, init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
@@ -372,27 +371,6 @@ def should_include_usage(
|
||||
return include_usage, include_continuous_usage
|
||||
|
||||
|
||||
def process_lora_modules(
|
||||
args_lora_modules: list[LoRAModulePath], default_mm_loras: dict[str, str] | None
|
||||
) -> list[LoRAModulePath]:
|
||||
from vllm.entrypoints.openai.models.serving import LoRAModulePath
|
||||
|
||||
lora_modules = args_lora_modules
|
||||
if default_mm_loras:
|
||||
default_mm_lora_paths = [
|
||||
LoRAModulePath(
|
||||
name=modality,
|
||||
path=lora_path,
|
||||
)
|
||||
for modality, lora_path in default_mm_loras.items()
|
||||
]
|
||||
if args_lora_modules is None:
|
||||
lora_modules = default_mm_lora_paths
|
||||
else:
|
||||
lora_modules += default_mm_lora_paths
|
||||
return lora_modules
|
||||
|
||||
|
||||
def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
|
||||
if envs.VLLM_DISABLE_LOG_LOGO or (formatter := current_formatter_type(lgr)) is None:
|
||||
message = "vLLM server version %s, serving model %s"
|
||||
|
||||
@@ -9,7 +9,6 @@ import torch
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
|
||||
@@ -41,7 +40,6 @@ class RequestLogger:
|
||||
prompt_token_ids: list[int] | None,
|
||||
prompt_embeds: torch.Tensor | None,
|
||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
max_log_len = self.max_log_len
|
||||
@@ -63,10 +61,9 @@ class RequestLogger:
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Received request %s: params: %s, lora_request: %s.",
|
||||
"Received request %s: params: %s.",
|
||||
request_id,
|
||||
params,
|
||||
lora_request,
|
||||
)
|
||||
|
||||
def log_request_body(self, request: AnyRequest) -> None:
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,12 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from typing import Literal, TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
## Protocols for Audio
|
||||
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
|
||||
TranscriptionResponseFormat: TypeAlias = AudioResponseFormat | Literal["diarized_json"]
|
||||
_LONG_INFO = torch.iinfo(torch.long)
|
||||
@@ -1,846 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import io
|
||||
import math
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import AsyncGenerator, Callable, Set
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import cached_property
|
||||
from typing import Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
from fastapi import Request
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import EncoderDecoderInput, EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import FlatLogprobs, Logprob
|
||||
from vllm.model_executor.models import SupportsTranscription
|
||||
from vllm.multimodal.audio import get_audio_duration, split_audio
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
|
||||
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterators
|
||||
|
||||
from ..transcription.protocol import (
|
||||
TranscriptionResponse,
|
||||
TranscriptionResponseDiarized,
|
||||
TranscriptionResponseStreamChoice,
|
||||
TranscriptionResponseVerbose,
|
||||
TranscriptionSegment,
|
||||
TranscriptionStreamResponse,
|
||||
)
|
||||
from ..translation.protocol import (
|
||||
TranslationResponse,
|
||||
TranslationResponseStreamChoice,
|
||||
TranslationResponseVerbose,
|
||||
TranslationSegment,
|
||||
TranslationStreamResponse,
|
||||
)
|
||||
|
||||
SpeechToTextResponse: TypeAlias = (
|
||||
TranscriptionResponse | TranslationResponse | TranscriptionResponseDiarized
|
||||
)
|
||||
SpeechToTextResponseVerbose: TypeAlias = (
|
||||
TranscriptionResponseVerbose | TranslationResponseVerbose
|
||||
)
|
||||
SpeechToTextSegment: TypeAlias = TranscriptionSegment | TranslationSegment
|
||||
T = TypeVar("T", bound=SpeechToTextResponse)
|
||||
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
|
||||
S = TypeVar("S", bound=SpeechToTextSegment)
|
||||
|
||||
|
||||
ResponseType: TypeAlias = (
|
||||
TranscriptionResponse
|
||||
| TranslationResponse
|
||||
| TranscriptionResponseVerbose
|
||||
| TranscriptionResponseDiarized
|
||||
| TranslationResponseVerbose
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def asr_inter_chunk_separator(
|
||||
language: str | None, no_space_languages: Set[str]
|
||||
) -> str:
|
||||
"""Space to insert between ASR text chunks for streaming and non-streaming join.
|
||||
|
||||
Languages in ``no_space_languages`` (e.g. Chinese, Japanese) use an empty
|
||||
separator; others use a single ASCII space.
|
||||
"""
|
||||
return "" if language and language.lower() in no_space_languages else " "
|
||||
|
||||
|
||||
class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
"""Base class for speech-to-text operations like transcription and
|
||||
translation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
task_type: Literal["transcribe", "translate"] = "transcribe",
|
||||
enable_force_include_usage: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
|
||||
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
||||
self.task_type: Final = task_type
|
||||
|
||||
self.asr_config = self.model_cls.get_speech_to_text_config(
|
||||
self.model_config, task_type
|
||||
)
|
||||
self.streaming_post_processor_cls = (
|
||||
self.model_cls.get_streaming_post_processor_cls()
|
||||
)
|
||||
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
|
||||
self.max_audio_decode_duration_s: int = envs.VLLM_MAX_AUDIO_DECODE_DURATION_S
|
||||
self.max_audio_decode_bytes: int = envs.VLLM_MAX_AUDIO_DECODE_BYTES
|
||||
if self.model_cls.supports_segment_timestamp:
|
||||
self.tokenizer = cast(
|
||||
PreTrainedTokenizerBase,
|
||||
get_tokenizer(
|
||||
tokenizer_name=self.model_config.tokenizer,
|
||||
tokenizer_mode=self.model_config.tokenizer_mode,
|
||||
),
|
||||
)
|
||||
|
||||
if self.default_sampling_params:
|
||||
logger.info(
|
||||
"Overwriting default completion sampling param with: %s",
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
# setup preprocess resources
|
||||
# we keep separate thread pool for frontend preprocessing instead
|
||||
# of reusing the one from Renderer which showed lower throughput
|
||||
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
|
||||
num_audio_preprocess_workers = envs.VLLM_MAX_AUDIO_PREPROCESS_WORKERS
|
||||
self._preprocess_executor = ThreadPoolExecutor(
|
||||
max_workers=num_audio_preprocess_workers,
|
||||
thread_name_prefix="stt-preprocess",
|
||||
)
|
||||
self._decode_and_chunk_speech_async = make_async_with_semaphore(
|
||||
self._decode_and_chunk_speech, executor=self._preprocess_executor
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def model_cls(self) -> type[SupportsTranscription]:
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
|
||||
model_cls = get_model_cls(self.model_config)
|
||||
return cast(type[SupportsTranscription], model_cls)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._preprocess_executor.shutdown(wait=False)
|
||||
|
||||
def _decode_and_chunk_speech(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
) -> tuple[list[np.ndarray], float]:
|
||||
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
|
||||
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
|
||||
# transparently falls back to ffmpeg via an in-memory fd.
|
||||
# NOTE resample to model SR here for efficiency. This is also a
|
||||
# pre-requisite for chunking, as it assumes Whisper SR.
|
||||
try:
|
||||
with io.BytesIO(audio_data) as buf:
|
||||
y, sr = load_audio(
|
||||
buf,
|
||||
sr=self.asr_config.sample_rate,
|
||||
mono=True,
|
||||
max_duration_s=self.max_audio_decode_duration_s,
|
||||
max_decode_bytes=self.max_audio_decode_bytes,
|
||||
)
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from exc
|
||||
|
||||
duration = get_audio_duration(y=y, sr=sr)
|
||||
do_split_audio = self.asr_config.allow_audio_chunking and (
|
||||
self.asr_config.max_audio_clip_s is not None
|
||||
and duration > self.asr_config.max_audio_clip_s
|
||||
)
|
||||
|
||||
if not do_split_audio:
|
||||
chunks = [y]
|
||||
else:
|
||||
assert self.asr_config.max_audio_clip_s is not None
|
||||
assert self.asr_config.min_energy_split_window_size is not None
|
||||
chunks = split_audio(
|
||||
audio_data=y,
|
||||
sample_rate=int(sr),
|
||||
max_clip_duration_s=self.asr_config.max_audio_clip_s,
|
||||
overlap_duration_s=self.asr_config.overlap_chunk_second,
|
||||
min_energy_window_size=self.asr_config.min_energy_split_window_size,
|
||||
)
|
||||
|
||||
return chunks, duration
|
||||
|
||||
async def _detect_language(
|
||||
self,
|
||||
audio_chunk: np.ndarray,
|
||||
request_id: str,
|
||||
) -> str:
|
||||
"""Auto-detect the spoken language from an audio chunk.
|
||||
|
||||
Delegates prompt construction and output parsing to the model class
|
||||
via ``get_language_detection_prompt`` and
|
||||
``parse_language_detection_output``.
|
||||
"""
|
||||
prompt = self.model_cls.get_language_detection_prompt(
|
||||
audio_chunk,
|
||||
self.asr_config,
|
||||
)
|
||||
allowed_token_ids = self.model_cls.get_language_token_ids(
|
||||
self.tokenizer,
|
||||
)
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
allowed_token_ids=allowed_token_ids,
|
||||
watermarking=False,
|
||||
)
|
||||
|
||||
result_generator = self.engine_client.generate(
|
||||
prompt,
|
||||
sampling_params,
|
||||
request_id,
|
||||
)
|
||||
|
||||
try:
|
||||
final_output: RequestOutput
|
||||
async for final_output in result_generator:
|
||||
if final_output.finished:
|
||||
break
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(request_id),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
token_ids = list(final_output.outputs[0].token_ids)
|
||||
lang = self.model_cls.parse_language_detection_output(
|
||||
token_ids,
|
||||
self.tokenizer,
|
||||
)
|
||||
|
||||
logger.info("Auto-detected language: '%s'", lang)
|
||||
return lang
|
||||
|
||||
async def _preprocess_speech_to_text(
|
||||
self,
|
||||
request: SpeechToTextRequest,
|
||||
audio_data: bytes,
|
||||
request_id: str,
|
||||
) -> tuple[list[EngineInput], float, list[float]]:
|
||||
# Validate request
|
||||
request.language = self.model_cls.validate_language(request.language)
|
||||
request.to_language = (
|
||||
self.model_cls.validate_language(request.to_language)
|
||||
if request.to_language
|
||||
else None
|
||||
)
|
||||
|
||||
# Run cpu intensive preprocess step in a separate thread pool executor.
|
||||
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
|
||||
|
||||
chunk_start_offsets: list[float] = [0.0]
|
||||
|
||||
for chunk in chunks[:-1]:
|
||||
chunk_start_offsets.append(
|
||||
chunk_start_offsets[-1] + chunk.shape[-1] / self.asr_config.sample_rate
|
||||
)
|
||||
|
||||
if request.language is None and getattr(
|
||||
self.model_cls, "supports_explicit_language_detection", False
|
||||
):
|
||||
# Auto-detect language from the first chunk.
|
||||
request.language = await self._detect_language(
|
||||
chunks[0], f"{request_id}-lang_detect"
|
||||
)
|
||||
|
||||
parsed_prompts: list[DictPrompt] = []
|
||||
for chunk in chunks:
|
||||
stt_params = request.build_stt_params(
|
||||
audio=chunk,
|
||||
stt_config=self.asr_config,
|
||||
model_config=self.model_config,
|
||||
task_type=self.task_type,
|
||||
)
|
||||
prompt = self.model_cls.get_generation_prompt(stt_params)
|
||||
|
||||
parsed_prompt: DictPrompt
|
||||
if request.response_format == "verbose_json":
|
||||
parsed_prompt = parse_enc_dec_prompt(prompt)
|
||||
parsed_prompt = self._preprocess_verbose_prompt(parsed_prompt)
|
||||
else:
|
||||
parsed_prompt = parse_model_prompt(self.model_config, prompt)
|
||||
|
||||
parsed_prompts.append(parsed_prompt)
|
||||
|
||||
engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)
|
||||
|
||||
return engine_inputs, duration, chunk_start_offsets
|
||||
|
||||
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
|
||||
dec_prompt = prompt["decoder_prompt"]
|
||||
|
||||
if not (isinstance(dec_prompt, dict) and "prompt" in dec_prompt):
|
||||
raise VLLMValidationError(
|
||||
"Expected decoder_prompt to contain text",
|
||||
parameter="decoder_prompt",
|
||||
value=type(dec_prompt).__name__,
|
||||
)
|
||||
|
||||
dec_prompt["prompt"] = dec_prompt["prompt"].replace(
|
||||
"<|notimestamps|>", "<|0.00|>"
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
@staticmethod
|
||||
def _get_decoder_prompt_len(engine_inputs: list[EngineInput]) -> int:
|
||||
"""Get the length of the decoder prompt. Currently we need to offset
|
||||
by the decoder prompt length when running beam search because the mm
|
||||
encoder is not currently cached and runs on decode calls; because of
|
||||
this, we need to make sure the redundant encoder calls won't exceed
|
||||
the context :(
|
||||
|
||||
FIXME (Alex) - this will be removed in the very near future once the
|
||||
encoder/decoder caching is implemented.
|
||||
"""
|
||||
input_len = 0
|
||||
assert len(engine_inputs) > 0
|
||||
first_input = engine_inputs[0]
|
||||
|
||||
if first_input.get("type") == "enc_dec":
|
||||
first_input = cast(EncoderDecoderInput, first_input)
|
||||
input_len = len(first_input["decoder_prompt"]["prompt_token_ids"])
|
||||
|
||||
return input_len
|
||||
|
||||
def _get_verbose_segments(
|
||||
self,
|
||||
tokens: tuple,
|
||||
log_probs: FlatLogprobs | list[dict[int, Logprob]],
|
||||
request: SpeechToTextRequest,
|
||||
segment_class: type[SpeechToTextSegment],
|
||||
start_time: float = 0,
|
||||
) -> list[SpeechToTextSegment]:
|
||||
"""Convert tokens to verbose segments.
|
||||
|
||||
This method expects the model to produce
|
||||
timestamps as tokens (similar to Whisper).
|
||||
If the tokens do not include timestamp information,
|
||||
the segments may not be generated correctly.
|
||||
|
||||
Note: No_speech_prob field is not supported
|
||||
in this implementation and will be None. See docs for details.
|
||||
"""
|
||||
BASE_OFFSET = 0.02
|
||||
init_token = self.tokenizer.encode("<|0.00|>", add_special_tokens=False)[0]
|
||||
if tokens[-1] == self.tokenizer.eos_token_id:
|
||||
tokens = tokens[:-1]
|
||||
|
||||
tokens_with_start = (init_token,) + tokens
|
||||
segments: list[SpeechToTextSegment] = []
|
||||
last_timestamp_start = 0
|
||||
|
||||
if tokens_with_start[-2] < init_token and tokens_with_start[-1] >= init_token:
|
||||
tokens_with_start = tokens_with_start + (tokens_with_start[-1],)
|
||||
avg_logprob = 0.0
|
||||
for idx in range(1, len(tokens_with_start)):
|
||||
# Timestamp tokens (e.g., <|0.00|>) are assumed to be sorted.
|
||||
# If the ordering is violated, this slicing may produce incorrect results.
|
||||
token = tokens_with_start[idx]
|
||||
if token >= init_token and tokens_with_start[idx - 1] >= init_token:
|
||||
sliced_timestamp_tokens = tokens_with_start[last_timestamp_start:idx]
|
||||
start_timestamp = sliced_timestamp_tokens[0] - init_token
|
||||
end_timestamp = sliced_timestamp_tokens[-1] - init_token
|
||||
text = self.tokenizer.decode(sliced_timestamp_tokens[1:-1])
|
||||
text_bytes = text.encode("utf-8")
|
||||
|
||||
casting_segment = cast(
|
||||
SpeechToTextSegment,
|
||||
segment_class(
|
||||
id=len(segments),
|
||||
seek=int(start_time),
|
||||
start=start_time + BASE_OFFSET * start_timestamp,
|
||||
end=start_time + BASE_OFFSET * end_timestamp,
|
||||
temperature=request.temperature,
|
||||
text=text,
|
||||
# The compression ratio measures
|
||||
# how compressible the generated text is.
|
||||
# A higher ratio indicates more repetitive content,
|
||||
# which is a strong sign of hallucination in outputs.
|
||||
compression_ratio=len(text_bytes)
|
||||
/ len(zlib.compress(text_bytes)),
|
||||
tokens=sliced_timestamp_tokens[1:-1],
|
||||
avg_logprob=avg_logprob / (idx - last_timestamp_start),
|
||||
),
|
||||
)
|
||||
segments.append(casting_segment)
|
||||
last_timestamp_start = idx
|
||||
avg_logprob = 0
|
||||
else:
|
||||
avg_logprob += log_probs[idx - 1][token].logprob
|
||||
return segments
|
||||
|
||||
async def _create_speech_to_text(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
request: SpeechToTextRequest,
|
||||
raw_request: Request,
|
||||
response_class: type[ResponseType],
|
||||
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
|
||||
) -> T | V | AsyncGenerator[str, None] | ErrorResponse:
|
||||
"""Base method for speech-to-text operations like transcription and
|
||||
translation."""
|
||||
if request.stream and request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Streaming is not currently supported with beam search"
|
||||
)
|
||||
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if not request.model:
|
||||
request.model = self.models.model_name()
|
||||
|
||||
self._preflight()
|
||||
|
||||
if request.response_format not in [
|
||||
"text",
|
||||
"json",
|
||||
"verbose_json",
|
||||
"diarized_json",
|
||||
]:
|
||||
return self.create_error_response(
|
||||
"Currently only support response_format: "
|
||||
"`text`, `json`, `verbose_json` or `diarized_json`"
|
||||
)
|
||||
|
||||
if (
|
||||
request.response_format == "verbose_json"
|
||||
and not self.model_cls.supports_segment_timestamp
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"Currently do not support verbose_json for {request.model}"
|
||||
)
|
||||
|
||||
if (
|
||||
request.response_format == "diarized_json"
|
||||
and not self.model_cls.supports_diarized_transcription
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"Currently do not support diarized_json for {request.model}"
|
||||
)
|
||||
|
||||
if (
|
||||
request.response_format in {"verbose_json", "diarized_json"}
|
||||
and request.stream
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"{request.response_format} format doesn't support streaming case"
|
||||
)
|
||||
request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
|
||||
|
||||
request_metadata = RequestResponseMetadata(request_id=request_id)
|
||||
if raw_request:
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
(
|
||||
engine_inputs,
|
||||
duration_s,
|
||||
chunk_start_offsets,
|
||||
) = await self._preprocess_speech_to_text(
|
||||
request=request,
|
||||
audio_data=audio_data,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
max_model_len = self.model_config.max_model_len
|
||||
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
|
||||
|
||||
input_len = (
|
||||
SpeechToTextBaseServing._get_decoder_prompt_len(engine_inputs)
|
||||
if request.use_beam_search
|
||||
else 0
|
||||
)
|
||||
|
||||
# Unlike most decoder-only models, whisper generation length is not
|
||||
# constrained by the size of the input audio, which is mapped to a
|
||||
# fixed-size log-mel-spectogram. Still, allow for fewer tokens to be
|
||||
# generated by respecting the extra completion tokens arg.
|
||||
max_tokens = get_max_tokens(
|
||||
max_model_len,
|
||||
request.max_completion_tokens,
|
||||
input_len,
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
if request.use_beam_search:
|
||||
sampling_params = request.to_beam_search_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
)
|
||||
else:
|
||||
sampling_params = request.to_sampling_params(
|
||||
max_tokens,
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
if request.response_format == "verbose_json":
|
||||
sampling_params.logprobs = 1
|
||||
|
||||
engine_request_ids = [
|
||||
request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}"
|
||||
for idx in range(len(engine_inputs))
|
||||
]
|
||||
list_result_generator = []
|
||||
try:
|
||||
for request_id_item, engine_input in zip(engine_request_ids, engine_inputs):
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_input,
|
||||
params=sampling_params,
|
||||
request_id=request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
else:
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
list_result_generator.append(generator)
|
||||
except asyncio.CancelledError:
|
||||
logger.info(
|
||||
"Request %s cancelled; aborting %d transcription engine request(s).",
|
||||
request_id,
|
||||
len(engine_request_ids),
|
||||
)
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(engine_request_ids),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
separator = asr_inter_chunk_separator(
|
||||
request.language, self.model_cls.no_space_languages
|
||||
)
|
||||
|
||||
if request.stream:
|
||||
return stream_generator_method(
|
||||
request,
|
||||
list_result_generator,
|
||||
request_id,
|
||||
request_metadata,
|
||||
duration_s,
|
||||
separator,
|
||||
)
|
||||
# Non-streaming response.
|
||||
try:
|
||||
assert list_result_generator is not None
|
||||
chunk_segment_parts: list[list[SpeechToTextSegment]] = [
|
||||
[] for _ in list_result_generator
|
||||
]
|
||||
chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator]
|
||||
segments_types: dict[str, type[SpeechToTextSegment]] = {
|
||||
"transcribe": TranscriptionSegment,
|
||||
"translate": TranslationSegment,
|
||||
}
|
||||
segment_class: type[SpeechToTextSegment] = segments_types[self.task_type]
|
||||
chunk_size_in_s = self.asr_config.max_audio_clip_s
|
||||
if chunk_size_in_s is None:
|
||||
assert len(list_result_generator) == 1, (
|
||||
"`max_audio_clip_s` is set to None, audio cannot be chunked"
|
||||
)
|
||||
assert len(chunk_start_offsets) == len(list_result_generator)
|
||||
result_generator = merge_async_iterators(*list_result_generator)
|
||||
async for idx, op in result_generator:
|
||||
start_time = chunk_start_offsets[idx]
|
||||
if request.response_format == "verbose_json":
|
||||
assert op.outputs[0].logprobs
|
||||
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
|
||||
tokens=tuple(op.outputs[0].token_ids),
|
||||
segment_class=segment_class,
|
||||
request=request,
|
||||
start_time=start_time,
|
||||
log_probs=op.outputs[0].logprobs,
|
||||
)
|
||||
|
||||
chunk_segment_parts[idx].extend(segments)
|
||||
chunk_text_parts[idx].extend([seg.text for seg in segments])
|
||||
else:
|
||||
raw_text = op.outputs[0].text
|
||||
chunk_text_parts[idx].append(
|
||||
self.model_cls.post_process_output(raw_text)
|
||||
)
|
||||
total_segments = [
|
||||
segment
|
||||
for segment_parts in chunk_segment_parts
|
||||
for segment in segment_parts
|
||||
]
|
||||
text_parts = [text for text_part in chunk_text_parts for text in text_part]
|
||||
text = separator.join(text_parts)
|
||||
if self.task_type == "transcribe":
|
||||
final_response: ResponseType
|
||||
# add usage in TranscriptionResponse.
|
||||
usage = {
|
||||
"type": "duration",
|
||||
# rounded up as per openAI specs
|
||||
"seconds": int(math.ceil(duration_s)),
|
||||
}
|
||||
if request.response_format == "diarized_json":
|
||||
diarized_segments = self.model_cls.parse_diarized_transcript(text)
|
||||
if not diarized_segments:
|
||||
return self.create_error_response(
|
||||
"Model output did not contain a valid diarized transcript"
|
||||
)
|
||||
final_response = cast(
|
||||
T,
|
||||
TranscriptionResponseDiarized(
|
||||
duration=duration_s,
|
||||
text=separator.join(
|
||||
segment.text for segment in diarized_segments
|
||||
),
|
||||
segments=[
|
||||
{
|
||||
"id": f"seg_{index}",
|
||||
"start": segment.start,
|
||||
"end": segment.end,
|
||||
"text": segment.text,
|
||||
"speaker": segment.speaker,
|
||||
}
|
||||
for index, segment in enumerate(diarized_segments)
|
||||
],
|
||||
usage=usage,
|
||||
),
|
||||
)
|
||||
elif request.response_format != "verbose_json":
|
||||
final_response = cast(
|
||||
T, TranscriptionResponse(text=text, usage=usage)
|
||||
)
|
||||
else:
|
||||
final_response = cast(
|
||||
V,
|
||||
TranscriptionResponseVerbose(
|
||||
text=text,
|
||||
language=request.language,
|
||||
duration=duration_s,
|
||||
segments=total_segments,
|
||||
),
|
||||
)
|
||||
else:
|
||||
# no usage in response for translation task
|
||||
if request.response_format != "verbose_json":
|
||||
final_response = cast(T, TranslationResponse(text=text))
|
||||
else:
|
||||
final_response = cast(
|
||||
V,
|
||||
TranslationResponseVerbose(
|
||||
text=text,
|
||||
language=request.language,
|
||||
duration=duration_s,
|
||||
segments=total_segments,
|
||||
),
|
||||
)
|
||||
return final_response
|
||||
except asyncio.CancelledError:
|
||||
logger.info(
|
||||
"Request %s cancelled; aborting %d transcription engine request(s).",
|
||||
request_id,
|
||||
len(engine_request_ids),
|
||||
)
|
||||
await asyncio.gather(
|
||||
self.engine_client.abort(engine_request_ids),
|
||||
return_exceptions=True,
|
||||
)
|
||||
raise
|
||||
|
||||
async def _speech_to_text_stream_generator(
|
||||
self,
|
||||
request: SpeechToTextRequest,
|
||||
list_result_generator: list[AsyncGenerator[RequestOutput, None]],
|
||||
request_id: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
audio_duration_s: float,
|
||||
chunk_object_type: Literal["translation.chunk", "transcription.chunk"],
|
||||
response_stream_choice_class: type[TranscriptionResponseStreamChoice]
|
||||
| type[TranslationResponseStreamChoice],
|
||||
stream_response_class: type[TranscriptionStreamResponse]
|
||||
| type[TranslationStreamResponse],
|
||||
separator: str,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
created_time = int(time.time())
|
||||
model_name = request.model
|
||||
|
||||
completion_tokens = 0
|
||||
num_prompt_tokens = 0
|
||||
|
||||
include_usage = self.enable_force_include_usage or request.stream_include_usage
|
||||
include_continuous_usage = (
|
||||
request.stream_continuous_usage_stats
|
||||
if include_usage and request.stream_continuous_usage_stats
|
||||
else False
|
||||
)
|
||||
|
||||
try:
|
||||
for result_generator in list_result_generator:
|
||||
beginning_of_chunk = True
|
||||
post_processor = self.streaming_post_processor_cls()
|
||||
async for res in result_generator:
|
||||
# On first result.
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
if audio_tokens := self.model_cls.get_num_audio_tokens(
|
||||
audio_duration_s, self.asr_config, self.model_config
|
||||
):
|
||||
num_prompt_tokens += audio_tokens
|
||||
|
||||
# We need to do it here, because if there are exceptions in
|
||||
# the result_generator, it needs to be sent as the FIRST
|
||||
# response (by the try...catch).
|
||||
|
||||
# Just one output (n=1) supported.
|
||||
assert len(res.outputs) == 1
|
||||
output = res.outputs[0]
|
||||
|
||||
output_text = post_processor.process_delta(
|
||||
output.text, output.finish_reason is not None
|
||||
)
|
||||
|
||||
# dont add separator to the first chunk
|
||||
if (
|
||||
result_generator is not list_result_generator[0]
|
||||
and beginning_of_chunk
|
||||
and output_text
|
||||
):
|
||||
output_text = separator + output_text
|
||||
beginning_of_chunk = False
|
||||
|
||||
if output.finish_reason is None and not output_text:
|
||||
completion_tokens += len(output.token_ids)
|
||||
continue
|
||||
|
||||
delta_message = DeltaMessage(content=output_text)
|
||||
completion_tokens += len(output.token_ids)
|
||||
|
||||
if output.finish_reason is None:
|
||||
# Still generating, send delta update.
|
||||
choice_data = response_stream_choice_class(delta=delta_message)
|
||||
else:
|
||||
# Model is finished generating.
|
||||
choice_data = response_stream_choice_class(
|
||||
delta=delta_message,
|
||||
finish_reason=output.finish_reason,
|
||||
stop_reason=output.stop_reason,
|
||||
)
|
||||
|
||||
chunk = stream_response_class(
|
||||
id=request_id,
|
||||
object=chunk_object_type,
|
||||
created=created_time,
|
||||
choices=[choice_data],
|
||||
model=model_name,
|
||||
)
|
||||
|
||||
# handle usage stats if requested & if continuous
|
||||
if include_continuous_usage:
|
||||
chunk.usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=num_prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
|
||||
# Once the final token is handled, if stream_options.include_usage
|
||||
# is sent, send the usage.
|
||||
if include_usage:
|
||||
final_usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=num_prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
final_usage_chunk = stream_response_class(
|
||||
id=request_id,
|
||||
object=chunk_object_type,
|
||||
created=created_time,
|
||||
choices=[],
|
||||
model=model_name,
|
||||
usage=final_usage,
|
||||
)
|
||||
final_usage_data = final_usage_chunk.model_dump_json(
|
||||
exclude_unset=True, exclude_none=True
|
||||
)
|
||||
yield f"data: {final_usage_data}\n\n"
|
||||
|
||||
# report to FastAPI middleware aggregate usage across all choices
|
||||
request_metadata.final_usage_info = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=num_prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Error in %s stream generator.", self.task_type)
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
# Send the final done message after all response.n are finished
|
||||
yield "data: [DONE]\n\n"
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared utilities for speech-to-text API routes."""
|
||||
|
||||
from fastapi import UploadFile
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
|
||||
|
||||
_READ_CHUNK_SIZE = 64 * KiB_bytes
|
||||
|
||||
|
||||
async def read_upload_with_limit(
|
||||
file: UploadFile,
|
||||
max_size_mb: float | None = None,
|
||||
) -> bytes:
|
||||
"""Read an uploaded file enforcing a size limit *before* full
|
||||
materialization.
|
||||
|
||||
The function first checks the Content-Length header (``file.size``) when
|
||||
available. Regardless, it then performs a chunked read that stops as soon
|
||||
as the accumulated bytes exceed the limit, ensuring that an oversized
|
||||
upload never fully materializes in memory.
|
||||
|
||||
Args:
|
||||
file: The FastAPI/Starlette ``UploadFile`` object.
|
||||
max_size_mb: Maximum allowed compressed file size in megabytes.
|
||||
Defaults to ``envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB``.
|
||||
|
||||
Returns:
|
||||
The file content as ``bytes``.
|
||||
|
||||
Raises:
|
||||
VLLMValidationError: If the file exceeds the configured size limit.
|
||||
|
||||
"""
|
||||
if max_size_mb is None:
|
||||
max_size_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
|
||||
|
||||
max_bytes = int(max_size_mb * MiB_bytes)
|
||||
|
||||
if file.size is not None and file.size > max_bytes:
|
||||
raise VLLMValidationError(
|
||||
"Maximum file size exceeded",
|
||||
parameter="audio_filesize_mb",
|
||||
value=file.size / MiB_bytes,
|
||||
)
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
chunk = await file.read(_READ_CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise VLLMValidationError(
|
||||
"Maximum file size exceeded",
|
||||
parameter="audio_filesize_mb",
|
||||
value=total / MiB_bytes,
|
||||
)
|
||||
chunks.append(chunk)
|
||||
|
||||
return b"".join(chunks)
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.tasks import SupportedTask
|
||||
else:
|
||||
RequestLogger = object
|
||||
|
||||
|
||||
def register_speech_to_text_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
if "realtime" in supported_tasks:
|
||||
from .realtime.api_router import router as realtime_router
|
||||
|
||||
app.include_router(realtime_router)
|
||||
|
||||
if "transcription" in supported_tasks:
|
||||
from .transcription.api_router import router as transcription_router
|
||||
|
||||
app.include_router(transcription_router)
|
||||
|
||||
from .translation.api_router import router as translation_router
|
||||
|
||||
app.include_router(translation_router)
|
||||
|
||||
|
||||
def init_speech_to_text_state(
|
||||
engine_client: "EngineClient",
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
if "transcription" in supported_tasks:
|
||||
from .transcription.serving import OpenAIServingTranscription
|
||||
|
||||
state.openai_serving_transcription = OpenAIServingTranscription(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
)
|
||||
|
||||
from .translation.serving import OpenAIServingTranslation
|
||||
|
||||
state.openai_serving_translation = OpenAIServingTranslation(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
)
|
||||
|
||||
if "realtime" in supported_tasks:
|
||||
from .realtime.serving import OpenAIServingRealtime
|
||||
|
||||
state.openai_serving_realtime = OpenAIServingRealtime(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from fastapi import APIRouter, WebSocket
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .connection import RealtimeConnection
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.websocket("/v1/realtime")
|
||||
async def realtime_endpoint(websocket: WebSocket):
|
||||
"""WebSocket endpoint for realtime audio transcription.
|
||||
|
||||
Protocol:
|
||||
1. Client connects to ws://host/v1/realtime
|
||||
2. Server sends session.created event
|
||||
3. Client optionally sends session.update with model/params
|
||||
4. Client sends input_audio_buffer.commit when ready
|
||||
5. Client sends input_audio_buffer.append events with base64 PCM16 chunks
|
||||
6. Server processes and sends transcription.delta events
|
||||
7. Server sends transcription.done with final text + usage
|
||||
8. Repeat from step 5 for next utterance
|
||||
9. Optionally, client sends input_audio_buffer.commit with final=True
|
||||
to signal audio input is finished. Useful when streaming audio files
|
||||
|
||||
Audio format: PCM16, 16kHz, mono, base64-encoded
|
||||
"""
|
||||
app = websocket.app
|
||||
serving = app.state.openai_serving_realtime
|
||||
|
||||
connection = RealtimeConnection(websocket, serving)
|
||||
await connection.handle_connection()
|
||||
@@ -1,314 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from http import HTTPStatus
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from fastapi import WebSocket
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import (
|
||||
ErrorEvent,
|
||||
InputAudioBufferAppend,
|
||||
InputAudioBufferCommit,
|
||||
SessionCreated,
|
||||
TranscriptionDelta,
|
||||
TranscriptionDone,
|
||||
)
|
||||
from .serving import OpenAIServingRealtime
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RealtimeConnection:
|
||||
"""Manages WebSocket lifecycle and state for realtime transcription.
|
||||
|
||||
This class handles:
|
||||
- WebSocket connection lifecycle (accept, receive, send, close)
|
||||
- Event routing (session.update, append, commit)
|
||||
- Audio buffering via asyncio.Queue
|
||||
- Generation task management
|
||||
- Error handling and cleanup
|
||||
"""
|
||||
|
||||
def __init__(self, websocket: WebSocket, serving: OpenAIServingRealtime):
|
||||
self.websocket = websocket
|
||||
self.connection_id = f"ws-{uuid4()}"
|
||||
self.serving = serving
|
||||
self.audio_queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
|
||||
self.generation_task: asyncio.Task | None = None
|
||||
self.request_id: str | None = None
|
||||
|
||||
self._is_connected = False
|
||||
self._is_model_validated = False
|
||||
|
||||
self._max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
|
||||
|
||||
async def handle_connection(self):
|
||||
"""Main connection loop."""
|
||||
await self.websocket.accept()
|
||||
logger.debug("WebSocket connection accepted: %s", self.connection_id)
|
||||
self._is_connected = True
|
||||
|
||||
# Send session created event
|
||||
await self.send(SessionCreated())
|
||||
|
||||
try:
|
||||
while True:
|
||||
message = await self.websocket.receive_text()
|
||||
try:
|
||||
event = json.loads(message)
|
||||
await self.handle_event(event)
|
||||
except json.JSONDecodeError:
|
||||
await self.send_error("Invalid JSON", "invalid_json")
|
||||
except Exception as e:
|
||||
logger.exception("Error handling event: %s", e)
|
||||
await self.send_error(sanitize_message(str(e)), "processing_error")
|
||||
except WebSocketDisconnect:
|
||||
logger.debug("WebSocket disconnected: %s", self.connection_id)
|
||||
self._is_connected = False
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error in connection: %s", e)
|
||||
finally:
|
||||
await self.cleanup()
|
||||
|
||||
def _check_model(self, model: str | None) -> None | ErrorResponse:
|
||||
if self.serving._is_model_supported(model):
|
||||
return None
|
||||
|
||||
return self.serving.create_error_response(
|
||||
message=f"The model `{model}` does not exist.",
|
||||
err_type="NotFoundError",
|
||||
status_code=HTTPStatus.NOT_FOUND,
|
||||
param="model",
|
||||
)
|
||||
|
||||
async def handle_event(self, event: dict):
|
||||
"""Route events to handlers.
|
||||
|
||||
Supported event types:
|
||||
- session.update: Configure model
|
||||
- input_audio_buffer.append: Add audio chunk to queue
|
||||
- input_audio_buffer.commit: Start transcription generation
|
||||
"""
|
||||
event_type = event.get("type")
|
||||
if event_type == "session.update":
|
||||
logger.debug("Session updated: %s", event)
|
||||
model = event.get("model")
|
||||
if model is None:
|
||||
await self.send_error("Missing required field: model", "invalid_event")
|
||||
return
|
||||
err = self._check_model(model)
|
||||
if err is not None:
|
||||
await self.send_error(err.error.message, "model_not_found")
|
||||
return
|
||||
self._is_model_validated = True
|
||||
elif event_type == "input_audio_buffer.append":
|
||||
append_event = InputAudioBufferAppend(**event)
|
||||
try:
|
||||
audio_bytes = base64.b64decode(append_event.audio)
|
||||
# Convert PCM16 bytes to float32 numpy array
|
||||
audio_array = (
|
||||
np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
|
||||
/ 32768.0
|
||||
)
|
||||
|
||||
if len(audio_array) / 1024**2 > self._max_audio_filesize_mb:
|
||||
raise VLLMValidationError(
|
||||
"Maximum file size exceeded",
|
||||
parameter="audio_filesize_mb",
|
||||
value=len(audio_array) / 1024**2,
|
||||
)
|
||||
if len(audio_array) == 0:
|
||||
raise VLLMValidationError("Can't process empty audio.")
|
||||
|
||||
# Put audio chunk in queue
|
||||
self.audio_queue.put_nowait(audio_array)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to decode audio: %s", e)
|
||||
await self.send_error("Invalid audio data", "invalid_audio")
|
||||
|
||||
elif event_type == "input_audio_buffer.commit":
|
||||
if not self._is_model_validated:
|
||||
err_msg = (
|
||||
"Model not validated. Make sure to validate the"
|
||||
" model by sending a session.update event."
|
||||
)
|
||||
await self.send_error(
|
||||
err_msg,
|
||||
"model_not_validated",
|
||||
)
|
||||
return
|
||||
|
||||
commit_event = InputAudioBufferCommit(**event)
|
||||
# final signals that the audio is finished
|
||||
if commit_event.final:
|
||||
self.audio_queue.put_nowait(None)
|
||||
else:
|
||||
await self.start_generation()
|
||||
else:
|
||||
await self.send_error(f"Unknown event type: {event_type}", "unknown_event")
|
||||
|
||||
async def audio_stream_generator(self) -> AsyncGenerator[np.ndarray, None]:
|
||||
"""Generator that yields audio chunks from the queue."""
|
||||
while True:
|
||||
audio_chunk = await self.audio_queue.get()
|
||||
if audio_chunk is None: # Sentinel value to stop
|
||||
break
|
||||
yield audio_chunk
|
||||
|
||||
async def start_generation(self):
|
||||
"""Start the transcription generation task."""
|
||||
if self.generation_task is not None and not self.generation_task.done():
|
||||
logger.warning("Generation already in progress, ignoring commit")
|
||||
return
|
||||
|
||||
# Create audio stream generator
|
||||
audio_stream = self.audio_stream_generator()
|
||||
input_stream = asyncio.Queue[list[int]]()
|
||||
|
||||
# Transform to StreamingInput generator
|
||||
streaming_input_gen = self.serving.transcribe_realtime(
|
||||
audio_stream, input_stream
|
||||
)
|
||||
|
||||
# Start generation task
|
||||
self.generation_task = asyncio.create_task(
|
||||
self._run_generation(streaming_input_gen, input_stream)
|
||||
)
|
||||
|
||||
async def _run_generation(
|
||||
self,
|
||||
streaming_input_gen: AsyncGenerator,
|
||||
input_stream: asyncio.Queue[list[int]],
|
||||
):
|
||||
"""Run the generation and stream results back to the client.
|
||||
|
||||
This method:
|
||||
1. Creates sampling parameters from session config
|
||||
2. Passes the streaming input generator to engine.generate()
|
||||
3. Streams transcription.delta events as text is generated
|
||||
4. Sends final transcription.done event with usage stats
|
||||
5. Feeds generated token IDs back to input_stream for next iteration
|
||||
6. Cleans up the audio queue
|
||||
"""
|
||||
request_id = f"rt-{self.connection_id}-{uuid4()}"
|
||||
self.request_id = request_id
|
||||
full_text = ""
|
||||
|
||||
prompt_token_ids_len: int = 0
|
||||
completion_tokens_len: int = 0
|
||||
|
||||
try:
|
||||
# Create sampling params
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
|
||||
sampling_params = SamplingParams.from_optional(
|
||||
temperature=0.0,
|
||||
max_tokens=self.serving.model_cls.realtime_max_tokens,
|
||||
output_kind=RequestOutputKind.DELTA,
|
||||
skip_clone=True,
|
||||
)
|
||||
|
||||
# Pass the streaming input generator to the engine
|
||||
# The engine will consume audio chunks as they arrive and
|
||||
# stream back transcription results incrementally
|
||||
result_gen = self.serving.engine_client.generate(
|
||||
prompt=streaming_input_gen,
|
||||
sampling_params=sampling_params,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
# Stream results back to client as they're generated
|
||||
async for output in result_gen:
|
||||
if output.outputs and len(output.outputs) > 0:
|
||||
if not prompt_token_ids_len and output.prompt_token_ids:
|
||||
prompt_token_ids_len = len(output.prompt_token_ids)
|
||||
|
||||
delta = output.outputs[0].text
|
||||
full_text += delta
|
||||
|
||||
# append output to input
|
||||
input_stream.put_nowait(list(output.outputs[0].token_ids))
|
||||
await self.send(TranscriptionDelta(delta=delta))
|
||||
|
||||
completion_tokens_len += len(output.outputs[0].token_ids)
|
||||
|
||||
if not self._is_connected:
|
||||
# finish because websocket connection was killed
|
||||
break
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_token_ids_len,
|
||||
completion_tokens=completion_tokens_len,
|
||||
total_tokens=prompt_token_ids_len + completion_tokens_len,
|
||||
)
|
||||
|
||||
# Send final completion event
|
||||
await self.send(TranscriptionDone(text=full_text, usage=usage))
|
||||
|
||||
# Clear queue for next utterance
|
||||
while not self.audio_queue.empty():
|
||||
self.audio_queue.get_nowait()
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Error in generation: %s", e)
|
||||
# The socket may already be closed; nothing more we can do.
|
||||
with contextlib.suppress(Exception):
|
||||
await self.send_error(sanitize_message(str(e)), "processing_error")
|
||||
|
||||
async def send(
|
||||
self, event: SessionCreated | TranscriptionDelta | TranscriptionDone
|
||||
):
|
||||
"""Send event to client."""
|
||||
data = event.model_dump_json()
|
||||
await self.websocket.send_text(data)
|
||||
|
||||
async def send_error(self, message: str, code: str | None = None):
|
||||
"""Send error event to client."""
|
||||
error_event = ErrorEvent(error=message, code=code)
|
||||
await self.websocket.send_text(error_event.model_dump_json())
|
||||
|
||||
async def cleanup(self):
|
||||
"""Cleanup resources."""
|
||||
# Signal audio stream to stop
|
||||
self.audio_queue.put_nowait(None)
|
||||
|
||||
# Abort the engine request. Cancelling the asyncio task alone does NOT
|
||||
# abort the underlying engine request: a request parked in the
|
||||
# VAD-waiting state keeps occupying a slot and generating into a now
|
||||
# dead websocket (an orphaned request that never frees and starves
|
||||
# other streams). Aborting also unblocks the generation task, which is
|
||||
# otherwise stuck awaiting the next engine output.
|
||||
if self.request_id is not None:
|
||||
try:
|
||||
await self.serving.engine_client.abort(self.request_id)
|
||||
except Exception:
|
||||
logger.exception("Failed to abort request %s", self.request_id)
|
||||
self.request_id = None
|
||||
|
||||
# Cancel the generation task if still running, and await it so the
|
||||
# cancellation (and any generator cleanup) actually completes.
|
||||
if self.generation_task and not self.generation_task.done():
|
||||
self.generation_task.cancel()
|
||||
try:
|
||||
await self.generation_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Error awaiting cancelled generation task")
|
||||
|
||||
logger.debug("Connection cleanup complete: %s", self.connection_id)
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
# Client -> Server Events
|
||||
|
||||
|
||||
class InputAudioBufferAppend(OpenAIBaseModel):
|
||||
"""Append audio chunk to buffer."""
|
||||
|
||||
type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append"
|
||||
audio: str # base64-encoded PCM16 @ 16kHz
|
||||
|
||||
|
||||
class InputAudioBufferCommit(OpenAIBaseModel):
|
||||
"""Process accumulated audio buffer."""
|
||||
|
||||
type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit"
|
||||
final: bool = False
|
||||
|
||||
|
||||
# Server -> Client Events
|
||||
class SessionUpdate(OpenAIBaseModel):
|
||||
"""Configure session parameters."""
|
||||
|
||||
type: Literal["session.update"] = "session.update"
|
||||
model: str | None = None
|
||||
|
||||
|
||||
class SessionCreated(OpenAIBaseModel):
|
||||
"""Connection established notification."""
|
||||
|
||||
type: Literal["session.created"] = "session.created"
|
||||
id: str = Field(default_factory=lambda: f"sess-{random_uuid()}")
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
|
||||
|
||||
class TranscriptionDelta(OpenAIBaseModel):
|
||||
"""Incremental transcription text."""
|
||||
|
||||
type: Literal["transcription.delta"] = "transcription.delta"
|
||||
delta: str # Incremental text
|
||||
|
||||
|
||||
class TranscriptionDone(OpenAIBaseModel):
|
||||
"""Final transcription with usage stats."""
|
||||
|
||||
type: Literal["transcription.done"] = "transcription.done"
|
||||
text: str # Complete transcription
|
||||
usage: UsageInfo | None = None
|
||||
|
||||
|
||||
class ErrorEvent(OpenAIBaseModel):
|
||||
"""Error notification."""
|
||||
|
||||
type: Literal["error"] = "error"
|
||||
error: str
|
||||
code: str | None = None
|
||||
@@ -1,89 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from functools import cached_property
|
||||
from typing import Literal, cast
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.engine.protocol import EngineClient, StreamingInput
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsRealtime
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIServingRealtime(GenerateBaseServing):
|
||||
"""Realtime audio transcription service via WebSocket streaming.
|
||||
|
||||
Provides streaming audio-to-text transcription by transforming audio chunks
|
||||
into StreamingInput objects that can be consumed by the engine.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.task_type: Literal["realtime"] = "realtime"
|
||||
|
||||
logger.info("OpenAIServingRealtime initialized for task: %s", self.task_type)
|
||||
|
||||
@cached_property
|
||||
def model_cls(self) -> type[SupportsRealtime]:
|
||||
"""Get the model class that supports transcription."""
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
|
||||
model_cls = get_model_cls(self.model_config)
|
||||
return cast(type[SupportsRealtime], model_cls)
|
||||
|
||||
async def transcribe_realtime(
|
||||
self,
|
||||
audio_stream: AsyncGenerator[np.ndarray, None],
|
||||
input_stream: asyncio.Queue[list[int]],
|
||||
) -> AsyncGenerator[StreamingInput, None]:
|
||||
"""Transform audio stream into StreamingInput for engine.generate().
|
||||
|
||||
Args:
|
||||
audio_stream: Async generator yielding float32 numpy audio arrays
|
||||
input_stream: Queue containing context token IDs from previous
|
||||
generation outputs. Used for autoregressive multi-turn
|
||||
processing where each generation's output becomes the context
|
||||
for the next iteration.
|
||||
|
||||
Yields:
|
||||
StreamingInput objects containing audio prompts for the engine
|
||||
|
||||
"""
|
||||
model_config = self.model_config
|
||||
renderer = self.renderer
|
||||
|
||||
# mypy is being stupid
|
||||
# TODO(Patrick) - fix this
|
||||
stream_input_iter = cast(
|
||||
AsyncGenerator[PromptType, None],
|
||||
self.model_cls.buffer_realtime_audio(
|
||||
audio_stream, input_stream, model_config
|
||||
),
|
||||
)
|
||||
|
||||
async for prompt in stream_input_iter:
|
||||
parsed_prompt = parse_model_prompt(model_config, prompt)
|
||||
(engine_input,) = await renderer.render_cmpl_async([parsed_prompt])
|
||||
|
||||
yield StreamingInput(prompt=engine_input)
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from http import HTTPStatus
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Form, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.base.utils import read_upload_with_limit
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from .protocol import TranscriptionRequest, TranscriptionResponseVariant
|
||||
from .serving import OpenAIServingTranscription
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def transcription(request: Request) -> OpenAIServingTranscription:
|
||||
return request.app.state.openai_serving_transcription
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/audio/transcriptions",
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_transcriptions(
|
||||
raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
|
||||
):
|
||||
handler = transcription(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Transcriptions API")
|
||||
|
||||
audio_data = await read_upload_with_limit(request.file)
|
||||
|
||||
generator = await handler.create_transcription(audio_data, request, raw_request)
|
||||
|
||||
if isinstance(generator, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=generator.model_dump(), status_code=generator.error.code
|
||||
)
|
||||
|
||||
elif isinstance(generator, TranscriptionResponseVariant):
|
||||
return JSONResponse(content=generator.model_dump())
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
@@ -1,430 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias
|
||||
|
||||
from fastapi import HTTPException, UploadFile
|
||||
from pydantic import (
|
||||
Field,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import (
|
||||
BeamSearchParams,
|
||||
RequestOutputKind,
|
||||
SamplingParams,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import _LONG_INFO, TranscriptionResponseFormat
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
|
||||
delta: DeltaMessage
|
||||
finish_reason: str | None = None
|
||||
stop_reason: int | str | None = None
|
||||
|
||||
|
||||
class TranscriptionStreamResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
|
||||
object: Literal["transcription.chunk"] = "transcription.chunk"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
choices: list[TranscriptionResponseStreamChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
|
||||
|
||||
class TranscriptionRequest(OpenAIBaseModel):
|
||||
# Ordered by official OpenAI API documentation
|
||||
# https://platform.openai.com/docs/api-reference/audio/createTranscription
|
||||
|
||||
file: UploadFile
|
||||
"""
|
||||
The audio file object (not file name) to transcribe, in one of these
|
||||
formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
|
||||
"""
|
||||
|
||||
model: str | None = None
|
||||
"""ID of the model to use.
|
||||
"""
|
||||
|
||||
language: str | None = None
|
||||
"""The language of the input audio.
|
||||
|
||||
Supplying the input language in
|
||||
[ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
|
||||
will improve accuracy and latency.
|
||||
"""
|
||||
|
||||
hotwords: str | None = None
|
||||
"""
|
||||
hotwords refers to a list of important words or phrases that the model
|
||||
should pay extra attention to during transcription.
|
||||
"""
|
||||
|
||||
prompt: str = Field(default="")
|
||||
"""An optional text to guide the model's style or continue a previous audio
|
||||
segment.
|
||||
|
||||
The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
|
||||
should match the audio language.
|
||||
"""
|
||||
|
||||
response_format: TranscriptionResponseFormat = Field(default="json")
|
||||
"""
|
||||
The format of the output, in one of these options: `json`, `text`, `srt`,
|
||||
`verbose_json`, or `vtt`.
|
||||
"""
|
||||
|
||||
## TODO (varun) : Support if set to 0, certain thresholds are met !!
|
||||
|
||||
timestamp_granularities: list[Literal["word", "segment"]] = Field(
|
||||
alias="timestamp_granularities[]", default=[]
|
||||
)
|
||||
"""The timestamp granularities to populate for this transcription.
|
||||
|
||||
`response_format` must be set `verbose_json` to use timestamp granularities.
|
||||
Either or both of these options are supported: `word`, or `segment`. Note:
|
||||
There is no additional latency for segment timestamps, but generating word
|
||||
timestamps incurs additional latency.
|
||||
"""
|
||||
|
||||
stream: bool | None = False
|
||||
"""When set, it will enable output to be streamed in a similar fashion
|
||||
as the Chat Completion endpoint.
|
||||
"""
|
||||
# --8<-- [start:transcription-extra-params]
|
||||
# Flattened stream option to simplify form data.
|
||||
stream_include_usage: bool | None = False
|
||||
stream_continuous_usage_stats: bool | None = False
|
||||
|
||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional request parameters with (list of) string or "
|
||||
"numeric values, used by custom extensions."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:transcription-extra-params]
|
||||
|
||||
to_language: str | None = None
|
||||
"""The language of the output audio we transcribe to.
|
||||
|
||||
Please note that this is not currently used by supported models at this
|
||||
time, but it is a placeholder for future use, matching translation api.
|
||||
"""
|
||||
|
||||
# --8<-- [start:transcription-sampling-params]
|
||||
use_beam_search: bool = False
|
||||
"""Whether or not beam search should be used."""
|
||||
|
||||
n: int = 1
|
||||
"""The number of beams to be used in beam search."""
|
||||
|
||||
length_penalty: float = 1.0
|
||||
"""Length penalty to be used for beam search."""
|
||||
|
||||
include_stop_str_in_output: bool = False
|
||||
"""Whether to include the stop strings in output text."""
|
||||
|
||||
temperature: float = Field(default=0.0)
|
||||
"""The sampling temperature, between 0 and 1.
|
||||
|
||||
Higher values like 0.8 will make the output more random, while lower values
|
||||
like 0.2 will make it more focused / deterministic. If set to 0, the model
|
||||
will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
|
||||
to automatically increase the temperature until certain thresholds are hit.
|
||||
"""
|
||||
|
||||
watermarking: bool | None = None
|
||||
"""Whether to apply the engine's configured watermark to this request."""
|
||||
|
||||
top_p: float | None = None
|
||||
"""Enables nucleus (top-p) sampling, where tokens are selected from the
|
||||
smallest possible set whose cumulative probability exceeds `p`.
|
||||
"""
|
||||
|
||||
top_k: int | None = None
|
||||
"""Limits sampling to the `k` most probable tokens at each step."""
|
||||
|
||||
min_p: float | None = None
|
||||
"""Filters out tokens with a probability lower than `min_p`, ensuring a
|
||||
minimum likelihood threshold during sampling.
|
||||
"""
|
||||
|
||||
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
|
||||
"""The seed to use for sampling."""
|
||||
|
||||
frequency_penalty: float | None = 0.0
|
||||
"""The frequency penalty to use for sampling."""
|
||||
|
||||
repetition_penalty: float | None = None
|
||||
"""The repetition penalty to use for sampling."""
|
||||
|
||||
presence_penalty: float | None = 0.0
|
||||
"""The presence penalty to use for sampling."""
|
||||
|
||||
max_completion_tokens: int | None = None
|
||||
"""The maximum number of tokens to generate."""
|
||||
# --8<-- [end:transcription-sampling-params]
|
||||
|
||||
# Default sampling parameters for transcription requests.
|
||||
_DEFAULT_SAMPLING_PARAMS: dict = {
|
||||
"repetition_penalty": 1.0,
|
||||
"temperature": 1.0,
|
||||
"top_p": 1.0,
|
||||
"top_k": 0,
|
||||
"min_p": 0.0,
|
||||
}
|
||||
|
||||
def build_stt_params(
|
||||
self,
|
||||
audio: "np.ndarray",
|
||||
stt_config: "SpeechToTextConfig",
|
||||
model_config: "ModelConfig",
|
||||
task_type: str,
|
||||
) -> SpeechToTextParams:
|
||||
return SpeechToTextParams(
|
||||
audio=audio,
|
||||
stt_config=stt_config,
|
||||
model_config=model_config,
|
||||
language=self.language,
|
||||
task_type=task_type,
|
||||
request_prompt=self.prompt,
|
||||
to_language=self.to_language,
|
||||
hotwords=self.hotwords,
|
||||
)
|
||||
|
||||
def to_beam_search_params(
|
||||
self,
|
||||
default_max_tokens: int,
|
||||
default_sampling_params: dict | None = None,
|
||||
) -> BeamSearchParams:
|
||||
if default_sampling_params is None:
|
||||
default_sampling_params = {}
|
||||
|
||||
max_tokens = default_max_tokens
|
||||
n = self.n if self.n is not None else 1
|
||||
|
||||
# NOTE: Temp 0 is a different fallback than completions
|
||||
if (temperature := self.temperature) is None:
|
||||
temperature = default_sampling_params.get("temperature", 0)
|
||||
|
||||
return BeamSearchParams(
|
||||
beam_width=n,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
watermarking=self.watermarking,
|
||||
length_penalty=self.length_penalty,
|
||||
include_stop_str_in_output=self.include_stop_str_in_output,
|
||||
)
|
||||
|
||||
def to_sampling_params(
|
||||
self, default_max_tokens: int, default_sampling_params: dict | None = None
|
||||
) -> SamplingParams:
|
||||
max_tokens = default_max_tokens
|
||||
|
||||
if default_sampling_params is None:
|
||||
default_sampling_params = {}
|
||||
|
||||
# Default parameters
|
||||
if (temperature := self.temperature) is None:
|
||||
temperature = default_sampling_params.get(
|
||||
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
|
||||
)
|
||||
if (top_p := self.top_p) is None:
|
||||
top_p = default_sampling_params.get(
|
||||
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
|
||||
)
|
||||
if (top_k := self.top_k) is None:
|
||||
top_k = default_sampling_params.get(
|
||||
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
|
||||
)
|
||||
if (min_p := self.min_p) is None:
|
||||
min_p = default_sampling_params.get(
|
||||
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
|
||||
)
|
||||
|
||||
if (repetition_penalty := self.repetition_penalty) is None:
|
||||
repetition_penalty = default_sampling_params.get(
|
||||
"repetition_penalty",
|
||||
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
|
||||
)
|
||||
|
||||
return SamplingParams.from_optional(
|
||||
temperature=temperature,
|
||||
watermarking=self.watermarking,
|
||||
max_tokens=max_tokens,
|
||||
seed=self.seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
min_p=min_p,
|
||||
frequency_penalty=self.frequency_penalty,
|
||||
repetition_penalty=repetition_penalty,
|
||||
presence_penalty=self.presence_penalty,
|
||||
output_kind=RequestOutputKind.DELTA
|
||||
if self.stream
|
||||
else RequestOutputKind.FINAL_ONLY,
|
||||
extra_args=self.vllm_xargs,
|
||||
skip_clone=True, # Created fresh per request, safe to skip clone
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def validate_transcription_request(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
if isinstance(data.get("file"), str):
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
detail="Expected 'file' to be a file-like object, not 'str'.",
|
||||
)
|
||||
|
||||
stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
|
||||
stream = data.get("stream", False)
|
||||
if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
|
||||
# Find which specific stream option was set
|
||||
invalid_param = next(
|
||||
(so for so in stream_opts if data.get(so, False)),
|
||||
"stream_include_usage",
|
||||
)
|
||||
raise VLLMValidationError(
|
||||
"Stream options can only be defined when `stream=True`.",
|
||||
parameter=invalid_param,
|
||||
)
|
||||
|
||||
# Parse vllm_xargs from JSON string (form data sends it as a string)
|
||||
xargs = data.get("vllm_xargs")
|
||||
if isinstance(xargs, str):
|
||||
try:
|
||||
data["vllm_xargs"] = json.loads(xargs)
|
||||
except json.JSONDecodeError as e:
|
||||
raise VLLMValidationError(
|
||||
f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
|
||||
parameter="vllm_xargs",
|
||||
) from e
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# Transcription response objects
|
||||
class TranscriptionUsageAudio(OpenAIBaseModel):
|
||||
type: Literal["duration"] = "duration"
|
||||
seconds: int
|
||||
|
||||
|
||||
class TranscriptionResponse(OpenAIBaseModel):
|
||||
text: str
|
||||
"""The transcribed text."""
|
||||
usage: TranscriptionUsageAudio
|
||||
|
||||
|
||||
class TranscriptionWord(OpenAIBaseModel):
|
||||
end: float
|
||||
"""End time of the word in seconds."""
|
||||
|
||||
start: float
|
||||
"""Start time of the word in seconds."""
|
||||
|
||||
word: str
|
||||
"""The text content of the word."""
|
||||
|
||||
|
||||
class TranscriptionSegment(OpenAIBaseModel):
|
||||
id: int
|
||||
"""Unique identifier of the segment."""
|
||||
|
||||
avg_logprob: float
|
||||
"""Average logprob of the segment.
|
||||
|
||||
If the value is lower than -1, consider the logprobs failed.
|
||||
"""
|
||||
|
||||
compression_ratio: float
|
||||
"""Compression ratio of the segment.
|
||||
|
||||
If the value is greater than 2.4, consider the compression failed.
|
||||
"""
|
||||
|
||||
end: float
|
||||
"""End time of the segment in seconds."""
|
||||
|
||||
no_speech_prob: float | None = None
|
||||
"""Probability of no speech in the segment.
|
||||
|
||||
If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
|
||||
this segment silent.
|
||||
"""
|
||||
|
||||
seek: int
|
||||
"""Seek offset of the segment."""
|
||||
|
||||
start: float
|
||||
"""Start time of the segment in seconds."""
|
||||
|
||||
temperature: float
|
||||
"""Temperature parameter used for generating the segment."""
|
||||
|
||||
text: str
|
||||
"""Text content of the segment."""
|
||||
|
||||
tokens: list[int]
|
||||
"""Array of token IDs for the text content."""
|
||||
|
||||
|
||||
class TranscriptionResponseVerbose(OpenAIBaseModel):
|
||||
duration: float
|
||||
"""The duration of the input audio."""
|
||||
|
||||
language: str
|
||||
"""The language of the input audio."""
|
||||
|
||||
text: str
|
||||
"""The transcribed text."""
|
||||
|
||||
segments: list[TranscriptionSegment] | None = None
|
||||
"""Segments of the transcribed text and their corresponding details."""
|
||||
|
||||
words: list[TranscriptionWord] | None = None
|
||||
"""Extracted words and their corresponding timestamps."""
|
||||
|
||||
|
||||
class TranscriptionDiarizedSegment(OpenAIBaseModel):
|
||||
"""A speaker-attributed transcription segment."""
|
||||
|
||||
type: Literal["transcript.text.segment"] = "transcript.text.segment"
|
||||
id: str
|
||||
start: float
|
||||
end: float
|
||||
text: str
|
||||
speaker: str
|
||||
|
||||
|
||||
class TranscriptionResponseDiarized(OpenAIBaseModel):
|
||||
"""OpenAI-compatible diarized transcription response."""
|
||||
|
||||
task: Literal["transcribe"] = "transcribe"
|
||||
duration: float
|
||||
text: str
|
||||
segments: list[TranscriptionDiarizedSegment]
|
||||
usage: TranscriptionUsageAudio
|
||||
|
||||
|
||||
TranscriptionResponseVariant: TypeAlias = (
|
||||
TranscriptionResponse | TranscriptionResponseVerbose | TranscriptionResponseDiarized
|
||||
)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from ..base.serving import SpeechToTextBaseServing
|
||||
from .protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
TranscriptionResponseDiarized,
|
||||
TranscriptionResponseStreamChoice,
|
||||
TranscriptionResponseVerbose,
|
||||
TranscriptionStreamResponse,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIServingTranscription(SpeechToTextBaseServing):
|
||||
"""Handles transcription requests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
task_type="transcribe",
|
||||
enable_force_include_usage=enable_force_include_usage,
|
||||
)
|
||||
|
||||
async def create_transcription(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
request: TranscriptionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> (
|
||||
TranscriptionResponse
|
||||
| TranscriptionResponseVerbose
|
||||
| TranscriptionResponseDiarized
|
||||
| AsyncGenerator[str, None]
|
||||
| ErrorResponse
|
||||
):
|
||||
"""Transcription API similar to OpenAI's API.
|
||||
|
||||
See https://platform.openai.com/docs/api-reference/audio/createTranscription
|
||||
for the API specification. This API mimics the OpenAI transcription API.
|
||||
"""
|
||||
return await self._create_speech_to_text(
|
||||
audio_data=audio_data,
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
response_class=(
|
||||
TranscriptionResponseVerbose
|
||||
if request.response_format == "verbose_json"
|
||||
else TranscriptionResponseDiarized
|
||||
if request.response_format == "diarized_json"
|
||||
else TranscriptionResponse
|
||||
),
|
||||
stream_generator_method=self.transcription_stream_generator,
|
||||
)
|
||||
|
||||
async def transcription_stream_generator(
|
||||
self,
|
||||
request: TranscriptionRequest,
|
||||
result_generator: list[AsyncGenerator[RequestOutput, None]],
|
||||
request_id: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
audio_duration_s: float,
|
||||
separator: str,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
generator = self._speech_to_text_stream_generator(
|
||||
request=request,
|
||||
list_result_generator=result_generator,
|
||||
request_id=request_id,
|
||||
request_metadata=request_metadata,
|
||||
audio_duration_s=audio_duration_s,
|
||||
chunk_object_type="transcription.chunk",
|
||||
response_stream_choice_class=TranscriptionResponseStreamChoice,
|
||||
stream_response_class=TranscriptionStreamResponse,
|
||||
separator=separator,
|
||||
)
|
||||
async for chunk in generator:
|
||||
yield chunk
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from http import HTTPStatus
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Form, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.base.utils import read_upload_with_limit
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import TranslationRequest, TranslationResponseVariant
|
||||
from .serving import OpenAIServingTranslation
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def translation(request: Request) -> OpenAIServingTranslation:
|
||||
return request.app.state.openai_serving_translation
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/audio/translations",
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_translations(
|
||||
request: Annotated[TranslationRequest, Form()], raw_request: Request
|
||||
):
|
||||
handler = translation(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Translations API")
|
||||
|
||||
audio_data = await read_upload_with_limit(request.file)
|
||||
|
||||
generator = await handler.create_translation(audio_data, request, raw_request)
|
||||
|
||||
if isinstance(generator, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=generator.model_dump(), status_code=generator.error.code
|
||||
)
|
||||
|
||||
elif isinstance(generator, TranslationResponseVariant):
|
||||
return JSONResponse(content=generator.model_dump())
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
@@ -1,386 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias
|
||||
|
||||
from fastapi import HTTPException, UploadFile
|
||||
from pydantic import (
|
||||
Field,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import (
|
||||
BeamSearchParams,
|
||||
RequestOutputKind,
|
||||
SamplingParams,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import _LONG_INFO, AudioResponseFormat
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TranslationResponseStreamChoice(OpenAIBaseModel):
|
||||
delta: DeltaMessage
|
||||
finish_reason: str | None = None
|
||||
stop_reason: int | str | None = None
|
||||
|
||||
|
||||
class TranslationStreamResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
|
||||
object: Literal["translation.chunk"] = "translation.chunk"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
choices: list[TranslationResponseStreamChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
|
||||
|
||||
class TranslationRequest(OpenAIBaseModel):
|
||||
# Ordered by official OpenAI API documentation
|
||||
# https://platform.openai.com/docs/api-reference/audio/createTranslation
|
||||
|
||||
file: UploadFile
|
||||
"""
|
||||
The audio file object (not file name) to translate, in one of these
|
||||
formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
|
||||
"""
|
||||
|
||||
model: str | None = None
|
||||
"""ID of the model to use.
|
||||
"""
|
||||
|
||||
prompt: str = Field(default="")
|
||||
"""An optional text to guide the model's style or continue a previous audio
|
||||
segment.
|
||||
|
||||
The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
|
||||
should match the audio language.
|
||||
"""
|
||||
|
||||
response_format: AudioResponseFormat = Field(default="json")
|
||||
"""
|
||||
The format of the output, in one of these options: `json`, `text`, `srt`,
|
||||
`verbose_json`, or `vtt`.
|
||||
"""
|
||||
|
||||
# --8<-- [start:translation-sampling-params]
|
||||
use_beam_search: bool = False
|
||||
"""Whether or not beam search should be used."""
|
||||
|
||||
n: int = 1
|
||||
"""The number of beams to be used in beam search."""
|
||||
|
||||
length_penalty: float = 1.0
|
||||
"""Length penalty to be used for beam search."""
|
||||
|
||||
include_stop_str_in_output: bool = False
|
||||
"""Whether to include the stop strings in output text."""
|
||||
|
||||
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
|
||||
"""The seed to use for sampling."""
|
||||
|
||||
temperature: float = Field(default=0.0)
|
||||
"""The sampling temperature, between 0 and 1.
|
||||
|
||||
Higher values like 0.8 will make the output more random, while lower values
|
||||
like 0.2 will make it more focused / deterministic. If set to 0, the model
|
||||
will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
|
||||
to automatically increase the temperature until certain thresholds are hit.
|
||||
"""
|
||||
|
||||
watermarking: bool | None = None
|
||||
"""Whether to apply the engine's configured watermark to this request."""
|
||||
|
||||
top_p: float | None = None
|
||||
"""Enables nucleus (top-p) sampling, where tokens are selected from the
|
||||
smallest possible set whose cumulative probability exceeds `p`.
|
||||
"""
|
||||
|
||||
top_k: int | None = None
|
||||
"""Limits sampling to the `k` most probable tokens at each step."""
|
||||
|
||||
min_p: float | None = None
|
||||
"""Filters out tokens with a probability lower than `min_p`, ensuring a
|
||||
minimum likelihood threshold during sampling.
|
||||
"""
|
||||
|
||||
frequency_penalty: float | None = 0.0
|
||||
"""The frequency penalty to use for sampling."""
|
||||
|
||||
repetition_penalty: float | None = None
|
||||
"""The repetition penalty to use for sampling."""
|
||||
|
||||
presence_penalty: float | None = 0.0
|
||||
"""The presence penalty to use for sampling."""
|
||||
# --8<-- [end:translation-sampling-params]
|
||||
|
||||
# --8<-- [start:translation-extra-params]
|
||||
language: str | None = None
|
||||
"""The language of the input audio we translate from.
|
||||
|
||||
Supplying the input language in
|
||||
[ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
|
||||
will improve accuracy.
|
||||
"""
|
||||
|
||||
hotwords: str | None = None
|
||||
"""
|
||||
hotwords refers to a list of important words or phrases that the model
|
||||
should pay extra attention to during transcription.
|
||||
"""
|
||||
|
||||
to_language: str | None = None
|
||||
"""The language of the input audio we translate to.
|
||||
|
||||
Please note that this is not supported by all models, refer to the specific
|
||||
model documentation for more details.
|
||||
For instance, Whisper only supports `to_language=en`.
|
||||
"""
|
||||
|
||||
stream: bool | None = False
|
||||
"""Custom field not present in the original OpenAI definition. When set,
|
||||
it will enable output to be streamed in a similar fashion as the Chat
|
||||
Completion endpoint.
|
||||
"""
|
||||
# Flattened stream option to simplify form data.
|
||||
stream_include_usage: bool | None = False
|
||||
stream_continuous_usage_stats: bool | None = False
|
||||
|
||||
max_completion_tokens: int | None = None
|
||||
"""The maximum number of tokens to generate."""
|
||||
|
||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional request parameters with (list of) string or "
|
||||
"numeric values, used by custom extensions."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:translation-extra-params]
|
||||
|
||||
# Default sampling parameters for translation requests.
|
||||
_DEFAULT_SAMPLING_PARAMS: dict = {
|
||||
"repetition_penalty": 1.0,
|
||||
"temperature": 0,
|
||||
"top_p": 1.0,
|
||||
"top_k": 0,
|
||||
"min_p": 0.0,
|
||||
}
|
||||
|
||||
def build_stt_params(
|
||||
self,
|
||||
audio: "np.ndarray",
|
||||
stt_config: "SpeechToTextConfig",
|
||||
model_config: "ModelConfig",
|
||||
task_type: str,
|
||||
) -> SpeechToTextParams:
|
||||
return SpeechToTextParams(
|
||||
audio=audio,
|
||||
stt_config=stt_config,
|
||||
model_config=model_config,
|
||||
language=self.language,
|
||||
task_type=task_type,
|
||||
request_prompt=self.prompt,
|
||||
to_language=self.to_language,
|
||||
hotwords=self.hotwords,
|
||||
)
|
||||
|
||||
def to_beam_search_params(
|
||||
self,
|
||||
default_max_tokens: int,
|
||||
default_sampling_params: dict | None = None,
|
||||
) -> BeamSearchParams:
|
||||
if default_sampling_params is None:
|
||||
default_sampling_params = {}
|
||||
|
||||
max_tokens = default_max_tokens
|
||||
n = self.n if self.n is not None else 1
|
||||
|
||||
# NOTE: Temp 0 is a different fallback than completions
|
||||
if (temperature := self.temperature) is None:
|
||||
temperature = default_sampling_params.get("temperature", 0)
|
||||
|
||||
return BeamSearchParams(
|
||||
beam_width=n,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
watermarking=self.watermarking,
|
||||
length_penalty=self.length_penalty,
|
||||
include_stop_str_in_output=self.include_stop_str_in_output,
|
||||
)
|
||||
|
||||
def to_sampling_params(
|
||||
self, default_max_tokens: int, default_sampling_params: dict | None = None
|
||||
) -> SamplingParams:
|
||||
max_tokens = default_max_tokens
|
||||
|
||||
if default_sampling_params is None:
|
||||
default_sampling_params = {}
|
||||
# Default parameters
|
||||
if (temperature := self.temperature) is None:
|
||||
temperature = default_sampling_params.get(
|
||||
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
|
||||
)
|
||||
if (top_p := self.top_p) is None:
|
||||
top_p = default_sampling_params.get(
|
||||
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
|
||||
)
|
||||
if (top_k := self.top_k) is None:
|
||||
top_k = default_sampling_params.get(
|
||||
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
|
||||
)
|
||||
if (min_p := self.min_p) is None:
|
||||
min_p = default_sampling_params.get(
|
||||
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
|
||||
)
|
||||
if (repetition_penalty := self.repetition_penalty) is None:
|
||||
repetition_penalty = default_sampling_params.get(
|
||||
"repetition_penalty",
|
||||
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
|
||||
)
|
||||
|
||||
return SamplingParams.from_optional(
|
||||
temperature=temperature,
|
||||
watermarking=self.watermarking,
|
||||
max_tokens=max_tokens,
|
||||
seed=self.seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
min_p=min_p,
|
||||
frequency_penalty=self.frequency_penalty,
|
||||
repetition_penalty=repetition_penalty,
|
||||
presence_penalty=self.presence_penalty,
|
||||
output_kind=RequestOutputKind.DELTA
|
||||
if self.stream
|
||||
else RequestOutputKind.FINAL_ONLY,
|
||||
extra_args=self.vllm_xargs,
|
||||
skip_clone=True, # Created fresh per request, safe to skip clone
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def validate_stream_options(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
if isinstance(data.get("file"), str):
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
detail="Expected 'file' to be a file-like object, not 'str'.",
|
||||
)
|
||||
stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
|
||||
stream = data.get("stream", False)
|
||||
if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
|
||||
# Find which specific stream option was set
|
||||
invalid_param = next(
|
||||
(so for so in stream_opts if data.get(so, False)),
|
||||
"stream_include_usage",
|
||||
)
|
||||
raise VLLMValidationError(
|
||||
"Stream options can only be defined when `stream=True`.",
|
||||
parameter=invalid_param,
|
||||
)
|
||||
|
||||
xargs = data.get("vllm_xargs")
|
||||
if isinstance(xargs, str):
|
||||
try:
|
||||
data["vllm_xargs"] = json.loads(xargs)
|
||||
except json.JSONDecodeError as e:
|
||||
raise VLLMValidationError(
|
||||
f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
|
||||
parameter="vllm_xargs",
|
||||
) from e
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# Translation response objects
|
||||
class TranslationResponse(OpenAIBaseModel):
|
||||
text: str
|
||||
"""The translated text."""
|
||||
|
||||
|
||||
class TranslationWord(OpenAIBaseModel):
|
||||
end: float
|
||||
"""End time of the word in seconds."""
|
||||
|
||||
start: float
|
||||
"""Start time of the word in seconds."""
|
||||
|
||||
word: str
|
||||
"""The text content of the word."""
|
||||
|
||||
|
||||
class TranslationSegment(OpenAIBaseModel):
|
||||
id: int
|
||||
"""Unique identifier of the segment."""
|
||||
|
||||
avg_logprob: float
|
||||
"""Average logprob of the segment.
|
||||
|
||||
If the value is lower than -1, consider the logprobs failed.
|
||||
"""
|
||||
|
||||
compression_ratio: float
|
||||
"""Compression ratio of the segment.
|
||||
|
||||
If the value is greater than 2.4, consider the compression failed.
|
||||
"""
|
||||
|
||||
end: float
|
||||
"""End time of the segment in seconds."""
|
||||
|
||||
no_speech_prob: float | None = None
|
||||
"""Probability of no speech in the segment.
|
||||
|
||||
If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
|
||||
this segment silent.
|
||||
"""
|
||||
|
||||
seek: int
|
||||
"""Seek offset of the segment."""
|
||||
|
||||
start: float
|
||||
"""Start time of the segment in seconds."""
|
||||
|
||||
temperature: float
|
||||
"""Temperature parameter used for generating the segment."""
|
||||
|
||||
text: str
|
||||
"""Text content of the segment."""
|
||||
|
||||
tokens: list[int]
|
||||
"""Array of token IDs for the text content."""
|
||||
|
||||
|
||||
class TranslationResponseVerbose(OpenAIBaseModel):
|
||||
duration: float
|
||||
"""The duration of the input audio."""
|
||||
|
||||
language: str
|
||||
"""The language of the input audio."""
|
||||
|
||||
text: str
|
||||
"""The translated text."""
|
||||
|
||||
segments: list[TranslationSegment] | None = None
|
||||
"""Segments of the translated text and their corresponding details."""
|
||||
|
||||
words: list[TranslationWord] | None = None
|
||||
"""Extracted words and their corresponding timestamps."""
|
||||
|
||||
|
||||
TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose
|
||||
@@ -1,97 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
from ..base.serving import SpeechToTextBaseServing
|
||||
from .protocol import (
|
||||
TranslationRequest,
|
||||
TranslationResponse,
|
||||
TranslationResponseStreamChoice,
|
||||
TranslationResponseVerbose,
|
||||
TranslationStreamResponse,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIServingTranslation(SpeechToTextBaseServing):
|
||||
"""Handles translation requests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
task_type="translate",
|
||||
enable_force_include_usage=enable_force_include_usage,
|
||||
)
|
||||
|
||||
async def create_translation(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
request: TranslationRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> (
|
||||
TranslationResponse
|
||||
| TranslationResponseVerbose
|
||||
| AsyncGenerator[str, None]
|
||||
| ErrorResponse
|
||||
):
|
||||
"""Translation API similar to OpenAI's API.
|
||||
|
||||
See https://platform.openai.com/docs/api-reference/audio/createTranslation
|
||||
for the API specification. This API mimics the OpenAI translation API.
|
||||
"""
|
||||
return await self._create_speech_to_text(
|
||||
audio_data=audio_data,
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
response_class=(
|
||||
TranslationResponseVerbose
|
||||
if request.response_format == "verbose_json"
|
||||
else TranslationResponse
|
||||
),
|
||||
stream_generator_method=self.translation_stream_generator,
|
||||
)
|
||||
|
||||
async def translation_stream_generator(
|
||||
self,
|
||||
request: TranslationRequest,
|
||||
result_generator: list[AsyncGenerator[RequestOutput, None]],
|
||||
request_id: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
audio_duration_s: float,
|
||||
separator: str,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
generator = self._speech_to_text_stream_generator(
|
||||
request=request,
|
||||
list_result_generator=result_generator,
|
||||
request_id=request_id,
|
||||
request_metadata=request_metadata,
|
||||
audio_duration_s=audio_duration_s,
|
||||
chunk_object_type="translation.chunk",
|
||||
response_stream_choice_class=TranslationResponseStreamChoice,
|
||||
stream_response_class=TranslationStreamResponse,
|
||||
separator=separator,
|
||||
)
|
||||
async for chunk in generator:
|
||||
yield chunk
|
||||
Reference in New Issue
Block a user