MOVZXandHinata
5aa2ac64f8
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
122 lines
3.9 KiB
Python
122 lines
3.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import logging
|
|
from collections.abc import Sequence
|
|
|
|
import torch
|
|
|
|
from vllm import envs
|
|
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
|
from vllm.logger import init_logger
|
|
from vllm.pooling_params import PoolingParams
|
|
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class RequestLogger:
|
|
def __init__(self, *, max_log_len: int | None) -> None:
|
|
self.max_log_len = max_log_len
|
|
|
|
if not logger.isEnabledFor(logging.INFO):
|
|
logger.warning_once(
|
|
"`--enable-log-requests` is set but "
|
|
"the minimum log level is higher than INFO. "
|
|
"No request information will be logged."
|
|
)
|
|
elif not logger.isEnabledFor(logging.DEBUG):
|
|
logger.info_once(
|
|
"`--enable-log-requests` is set but "
|
|
"the minimum log level is higher than DEBUG. "
|
|
"Only limited information will be logged to minimize overhead. "
|
|
"To view more details, set `--log-level DEBUG`."
|
|
)
|
|
|
|
def log_inputs(
|
|
self,
|
|
request_id: str,
|
|
prompt: str | None,
|
|
prompt_token_ids: list[int] | None,
|
|
prompt_embeds: torch.Tensor | None,
|
|
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
|
) -> None:
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
max_log_len = self.max_log_len
|
|
if max_log_len is not None:
|
|
if prompt is not None:
|
|
prompt = prompt[:max_log_len]
|
|
|
|
if prompt_token_ids is not None:
|
|
prompt_token_ids = prompt_token_ids[:max_log_len]
|
|
|
|
logger.debug(
|
|
"Request %s details: prompt: %r, "
|
|
"prompt_token_ids: %s, "
|
|
"prompt_embeds shape: %s.",
|
|
request_id,
|
|
prompt,
|
|
prompt_token_ids,
|
|
prompt_embeds.shape if prompt_embeds is not None else None,
|
|
)
|
|
|
|
logger.info(
|
|
"Received request %s: params: %s.",
|
|
request_id,
|
|
params,
|
|
)
|
|
|
|
def log_request_body(self, request: AnyRequest) -> None:
|
|
try:
|
|
level = logging.getLevelNamesMapping()[envs.VLLM_REQUEST_BODY_LOG_LEVEL]
|
|
except KeyError:
|
|
logger.warning_once(
|
|
"Invalid logging level for request content: %s. "
|
|
"Request JSON body will not be logged.",
|
|
envs.VLLM_REQUEST_BODY_LOG_LEVEL,
|
|
)
|
|
return
|
|
|
|
logger.log(
|
|
level,
|
|
"Request %s JSON body: %s",
|
|
getattr(request, "request_id", "N/A"),
|
|
request.model_dump_json(exclude_unset=True)[: self.max_log_len],
|
|
)
|
|
|
|
def log_outputs(
|
|
self,
|
|
request_id: str,
|
|
outputs: str,
|
|
output_token_ids: Sequence[int] | None,
|
|
finish_reason: str | None = None,
|
|
is_streaming: bool = False,
|
|
delta: bool = False,
|
|
) -> None:
|
|
max_log_len = self.max_log_len
|
|
if max_log_len is not None and outputs is not None:
|
|
outputs = outputs[:max_log_len]
|
|
|
|
stream_info = ""
|
|
if is_streaming:
|
|
stream_info = " (streaming delta)" if delta else " (streaming complete)"
|
|
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
if max_log_len is not None and output_token_ids is not None:
|
|
output_token_ids = list(output_token_ids)[:max_log_len]
|
|
|
|
logger.debug(
|
|
"Generated response %s%s details: output_token_ids: %s",
|
|
request_id,
|
|
stream_info,
|
|
output_token_ids,
|
|
)
|
|
|
|
logger.info(
|
|
"Generated response %s%s: output: %r, finish_reason: %s",
|
|
request_id,
|
|
stream_info,
|
|
outputs,
|
|
finish_reason,
|
|
)
|