[Refactor][Model Runner V2][Multimodal] Move the encoder-only path out of the shared runner (#53176)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -69,7 +69,6 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsTensors,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
@@ -175,7 +174,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
self.device = device
|
||||
self.dtype = self.model_config.dtype
|
||||
self.is_encoder_only = vllm_config.is_encoder_only
|
||||
self.kv_cache_dtype = self.dtype
|
||||
if self.cache_config.cache_dtype != "auto":
|
||||
# Quantized KV cache.
|
||||
@@ -499,8 +497,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return encoder_runner.get_encoder_timing_stats()
|
||||
|
||||
def get_kv_cache_spec(self):
|
||||
if self.is_encoder_only:
|
||||
return {}
|
||||
return get_kv_cache_spec(self.vllm_config)
|
||||
|
||||
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
@@ -646,9 +642,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
is_profile: bool = False,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
if self.is_encoder_only:
|
||||
empty = torch.empty(0, device=self.device)
|
||||
return empty, empty
|
||||
if skip_attn and not is_profile:
|
||||
raise ValueError(
|
||||
"skip_attn must only be True for initial memory profiling."
|
||||
@@ -809,12 +802,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dummy_mm_inputs, mm_budget
|
||||
)
|
||||
|
||||
if self.is_encoder_only:
|
||||
torch.accelerator.synchronize()
|
||||
self.reset_encoder_cache()
|
||||
gc.collect()
|
||||
return
|
||||
|
||||
hidden_states, sample_hidden_states = self._dummy_run(
|
||||
self.max_num_tokens, skip_attn=True, is_profile=True
|
||||
)
|
||||
@@ -851,9 +838,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture_model(self) -> int:
|
||||
if self.is_encoder_only:
|
||||
return 0
|
||||
|
||||
assert self.cudagraph_manager is not None
|
||||
capture_encoder = (
|
||||
self.model_state.supports_mm_inputs
|
||||
@@ -1575,26 +1559,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
with self.ec_connector.maybe_get_output(
|
||||
scheduler_output
|
||||
) as ec_connector_output:
|
||||
if self.is_encoder_only:
|
||||
# Encode and publish, nothing else: this instance runs no
|
||||
# language model, so the gather inside prepare_inputs_embeds
|
||||
# would build an inputs_embeds nobody reads -- and it
|
||||
# raises "Encoder cache miss" for any scheduled item this
|
||||
# instance did not encode, taking the engine down with it.
|
||||
self.model_state.execute_mm_encoder(scheduled_encoder_inputs)
|
||||
else:
|
||||
inputs_embeds = self.model_state.prepare_inputs_embeds(
|
||||
scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
inputs_embeds = self.model_state.prepare_inputs_embeds(
|
||||
scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
if inputs_embeds is not None and not requires_raw_input_tokens(self.model):
|
||||
input_ids = None
|
||||
|
||||
if self.is_encoder_only:
|
||||
return ModelRunnerOutput.with_ec_conn_output(
|
||||
make_empty_encoder_model_runner_output(scheduler_output),
|
||||
ec_connector_output,
|
||||
)
|
||||
|
||||
model_inputs = {
|
||||
"input_ids": input_ids,
|
||||
"positions": input_batch.positions,
|
||||
|
||||
Reference in New Issue
Block a user