Files
vLLM/vllm/v1/worker/mm_encoder_model_runner.py
MOVZXandHinata 5aa2ac64f8 [Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-19 21:06:19 +07:00

164 lines
6.1 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model runner for instances that only run the multi-modal encoder.
An encoder-only instance -- `--mm-encoder-only`, or the producer side of
encoder-cache disaggregation -- encodes the multi-modal items and publishes the
embeddings. It runs no language model: no KV cache, sampler, or decoder graphs.
"""
import time
from collections.abc import Iterator
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.logger import init_logger
from vllm.sequence import IntermediateTensors
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
from vllm.v1.kv_cache_interface import KVCacheSpec
from vllm.v1.outputs import (
ModelRunnerOutput,
make_empty_encoder_model_runner_output,
)
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from vllm.v1.worker.workspace import lock_workspace
if TYPE_CHECKING:
from vllm.v1.core.sched.output import SchedulerOutput
logger = init_logger(__name__)
class MMEncoderModelRunner(GPUModelRunner):
"""Encoder-only variant of the V2 GPU model runner."""
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
assert self.supports_mm_inputs, (
"An encoder-only instance must serve a multi-modal model."
)
assert self.dp_size == 1, "An encoder-only instance does not support DP."
depth = vllm_config.max_concurrent_batches
# Blocking (sleep) events: busy-polling the driver lock can make this
# rank a straggler under contention.
self._input_events = [torch.Event(blocking=True) for _ in range(depth)]
self._input_event_idx = 0
@contextmanager
def input_tensor_semaphore(self) -> Iterator[None]:
"""Guard the host writes into the pooled input buffers.
`UvaBufferPool` recycles a slot every `max_concurrent_batches` steps and
the device reads the pooled host buffers in place. A sampling step is
ordered by the device wait in `AsyncOutput.get_output()`; this one never
waits, so entering blocks until the step that last wrote these slots is
done with them and leaving records this step's last write.
"""
idx = self._input_event_idx
input_event = self._input_events[idx]
input_event.synchronize()
try:
yield
finally:
input_event.record()
self._input_event_idx = (idx + 1) % len(self._input_events)
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
return {}
@torch.inference_mode()
def capture_model(self, *, profile_only: bool = False) -> int:
encoder_runner = self.model_state.encoder_runner
if not encoder_runner.has_cudagraph():
return 0
start_time = time.perf_counter()
with freeze_gc_for_cudagraph_capture():
torch.accelerator.empty_cache()
start_free_memory = torch.accelerator.get_memory_info()[0]
encoder_runner.capture()
graph_memory = start_free_memory - torch.accelerator.get_memory_info()[0]
if not profile_only:
lock_workspace()
manager = encoder_runner.cudagraph_manager
assert manager is not None
logger.info(
"Encoder graph capturing finished in %.0f secs, took %.2f GiB (%d graphs)",
time.perf_counter() - start_time,
graph_memory / (1 << 30),
manager.get_num_graphs_to_capture(),
)
return graph_memory
def needs_cudagraph_capture(self) -> bool:
return False
def _dummy_run(
self, *args: Any, **kwargs: Any
) -> tuple[torch.Tensor, torch.Tensor]:
empty = torch.empty(0, device=self.device)
return empty, empty
def _dummy_sampler_run(self, hidden_states: torch.Tensor) -> None:
return
def _dummy_pooler_run(self, hidden_states: torch.Tensor) -> None:
return
def _no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
return self._merge_ec_connector_no_forward(
scheduler_output, self.kv_connector.no_forward(scheduler_output)
)
@torch.inference_mode()
def execute_model(
self,
scheduler_output: "SchedulerOutput",
intermediate_tensors: IntermediateTensors | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
is_profile: bool = False,
context_len: int = 0,
) -> ModelRunnerOutput:
assert not dummy_run, "An encoder-only instance runs no dummy batch."
with self.input_tensor_semaphore():
self.update_pp_decode_requests()
self.finish_requests(scheduler_output)
self.free_states(scheduler_output)
self.add_requests(scheduler_output)
self.update_requests(scheduler_output)
self.block_tables.apply_staged_writes()
if scheduler_output.total_num_scheduled_tokens == 0:
return self._no_forward(scheduler_output)
batch_req_state, _ = self.gather_batch_req_state(scheduler_output, False)
assert batch_req_state is not None
# No decoder graph or DP padding; encoder graphs run independently.
batch_desc = BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=batch_req_state.num_tokens,
num_reqs=None,
)
self.prepare_inputs(scheduler_output, batch_req_state, batch_desc)
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
with self.ec_connector.maybe_get_output(
scheduler_output
) as ec_connector_output:
self.model_state.execute_mm_encoder(scheduled_encoder_inputs)
return ModelRunnerOutput.with_ec_conn_output(
make_empty_encoder_model_runner_output(scheduler_output),
ec_connector_output,
)