Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
75 lines
2.9 KiB
Python
75 lines
2.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""One-token reads that return the logprobs of chosen label tokens."""
|
|
|
|
from collections.abc import AsyncGenerator, Mapping, Sequence
|
|
from dataclasses import dataclass
|
|
|
|
from vllm.engine.protocol import EngineClient
|
|
from vllm.exceptions import GenerationError
|
|
from vllm.inputs import EngineInput
|
|
from vllm.outputs import RequestOutput
|
|
from vllm.sampling_params import SamplingParams
|
|
from vllm.utils.async_utils import merge_async_iterators
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LabelRead:
|
|
result: RequestOutput
|
|
logprobs: list[float]
|
|
"""Full-vocabulary logprob of each label token, in ``logprob_token_ids``
|
|
order."""
|
|
|
|
|
|
async def next_token_label_reads(
|
|
engine_client: EngineClient,
|
|
engine_inputs: Sequence[EngineInput],
|
|
sampling_params: Sequence[SamplingParams],
|
|
request_id: str,
|
|
*,
|
|
trace_headers: Mapping[str, str] | None = None,
|
|
priority: int = 0,
|
|
) -> list[LabelRead]:
|
|
"""Runs one read per input at once, as request ``{request_id}-{i}``. Each
|
|
read's params set ``max_tokens=1`` and the label tokens as
|
|
``logprob_token_ids``. Raises GenerationError naming the item when a read has
|
|
no output or lacks a label's logprob."""
|
|
generators: list[AsyncGenerator[RequestOutput, None]] = [
|
|
engine_client.generate(
|
|
engine_input,
|
|
params,
|
|
f"{request_id}-{i}",
|
|
trace_headers=trace_headers,
|
|
priority=priority,
|
|
)
|
|
for i, (engine_input, params) in enumerate(zip(engine_inputs, sampling_params))
|
|
]
|
|
results: list[RequestOutput | None] = [None] * len(generators)
|
|
async for i, res in merge_async_iterators(*generators):
|
|
results[i] = res
|
|
|
|
reads = []
|
|
for i, (result, params) in enumerate(zip(results, sampling_params)):
|
|
if result is None:
|
|
raise GenerationError(f"Failed to generate result for item {i}")
|
|
if not result.outputs:
|
|
raise GenerationError(f"No output generated for item {i}")
|
|
output = result.outputs[0]
|
|
if output.finish_reason == "error":
|
|
raise GenerationError(f"Generation error for item {i}")
|
|
if not output.logprobs:
|
|
raise GenerationError(
|
|
f"No logprobs available for item {i}. "
|
|
"This might indicate an issue with logprobs configuration."
|
|
)
|
|
logprobs = output.logprobs[0]
|
|
label_ids = params.logprob_token_ids or []
|
|
missing = [t for t in label_ids if t not in logprobs]
|
|
if missing:
|
|
raise GenerationError(
|
|
f"Token IDs {missing} not found in logprobs for item {i}. "
|
|
"This might indicate the tokens are outside the model's vocabulary."
|
|
)
|
|
reads.append(LabelRead(result, [logprobs[t].logprob for t in label_ids]))
|
|
return reads
|