diff --git a/examples/disaggregated/disaggregated_encoder/disagg_1e1pd_example.sh b/examples/disaggregated/disaggregated_encoder/disagg_1e1pd_example.sh index ed752a38c6..4bf60ea66d 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_1e1pd_example.sh +++ b/examples/disaggregated/disaggregated_encoder/disagg_1e1pd_example.sh @@ -29,7 +29,7 @@ if [[ -z "${DEVICE_AFFINITY_ENV:-}" ]]; then fi EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache}" -TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-12000}" # wait_for_server timeout +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-300}" # wait_for_server timeout NUM_PROMPTS="${NUM_PROMPTS:-100}" # number of prompts to send in benchmark diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 70f111db6e..4cef46020b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -536,6 +536,24 @@ class VllmConfig: ] return hash_str + @property + def is_ec_producer_only(self) -> bool: + ec_config = self.ec_transfer_config + return ( + ec_config is not None + and ec_config.is_ec_producer + and not ec_config.is_ec_consumer + ) + + @property + def is_encoder_only(self) -> bool: + mm_config = ( + self.model_config.multimodal_config + if self.model_config is not None + else None + ) + return self.is_ec_producer_only or bool(mm_config and mm_config.mm_encoder_only) + @property def max_concurrent_batches(self) -> int: # PP requires PP-size concurrent batches to fill the pipeline. @@ -2246,10 +2264,6 @@ class VllmConfig: # Will be added by https://github.com/vllm-project/vllm/pull/35045 unsupported.append("KV sharing fast prefill") - if self.ec_transfer_config is not None: - # Will be added by https://github.com/vllm-project/vllm/pull/38390 - unsupported.append("EC transfer") - return unsupported def _validate_v2_model_runner(self) -> None: diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 5d17868b17..90fc104dc2 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -120,6 +120,8 @@ class BlockTables: self.num_blocks.np[i, req_index] = start + len(block_ids) def apply_staged_writes(self) -> None: + if self.num_kv_cache_groups == 0: + return if self.num_kv_cache_groups == 1: # Single group: write directly, skipping the per-write group lookup. self.block_tables[0].apply_write() @@ -138,6 +140,8 @@ class BlockTables: out: tuple[torch.Tensor, ...] | None = None, out_ptrs: torch.Tensor | None = None, ) -> tuple[torch.Tensor, ...]: + if self.num_kv_cache_groups == 0: + return () if out is None: out = tuple(self.input_block_tables) out_ptrs = self.input_block_table_ptrs @@ -179,6 +183,8 @@ class BlockTables: num_tokens_padded: int, out: torch.Tensor | None = None, ) -> torch.Tensor: + if self.num_kv_cache_groups == 0: + return (self.slot_mappings if out is None else out)[:, :num_tokens_padded] num_reqs = idx_mapping.shape[0] num_groups = self.num_kv_cache_groups slot_mappings = self.slot_mappings if out is None else out diff --git a/vllm/v1/worker/gpu/ec_connector.py b/vllm/v1/worker/gpu/ec_connector.py new file mode 100644 index 0000000000..825763b820 --- /dev/null +++ b/vllm/v1/worker/gpu/ec_connector.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Generator +from contextlib import contextmanager +from typing import TYPE_CHECKING + +import torch + +from vllm.config import VllmConfig +from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer +from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase +from vllm.v1.outputs import ECConnectorOutput + +if TYPE_CHECKING: + from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache + + +class ECConnector: + """EC connector interface used by the V2 GPU model runner.""" + + @contextmanager + def maybe_get_output( + self, scheduler_output: "SchedulerOutput" + ) -> Generator[ECConnectorOutput | None, None, None]: + yield None + + +class ActiveECConnector(ECConnector): + def __init__( + self, + vllm_config: VllmConfig, + encoder_cache: dict[str, torch.Tensor], + ) -> None: + self.encoder_cache = encoder_cache + self.save_new_caches = vllm_config.is_ec_producer_only + self.ec_connector = get_ec_transfer() + assert isinstance(self.ec_connector, ECConnectorBase) + + @contextmanager + def maybe_get_output( + self, scheduler_output: "SchedulerOutput" + ) -> Generator[ECConnectorOutput | None, None, None]: + if scheduler_output.ec_connector_metadata is None: + yield None + return + + output = ECConnectorOutput() + ec_connector = self.ec_connector + assert scheduler_output.ec_connector_metadata is not None + ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata) + + if ec_connector.is_consumer: + ec_connector.start_load_caches(self.encoder_cache) + + cached_hashes = set(self.encoder_cache) if self.save_new_caches else None + try: + yield output + if cached_hashes is not None: + for mm_hash in self.encoder_cache.keys() - cached_hashes: + ec_connector.save_caches( + encoder_cache=self.encoder_cache, mm_hash=mm_hash + ) + finally: + output.finished_sending, output.finished_recving = ( + ec_connector.get_finished(scheduler_output.finished_req_ids) + ) + ec_connector.clear_connector_metadata() + + +NO_OP_EC_CONNECTOR = ECConnector() + + +def get_ec_connector( + vllm_config: VllmConfig, + encoder_cache: "EncoderCache | None", +) -> ECConnector: + if ( + not has_ec_transfer() + or vllm_config.model_config.is_encoder_decoder + or encoder_cache is None + ): + return NO_OP_EC_CONNECTOR + + return ActiveECConnector(vllm_config, encoder_cache.encoder_outputs) diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index f0d9b7431f..df4edbe7a5 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -51,6 +51,8 @@ class EncoderRunner: mm_feature = mm_features[mm_input_id] if mm_feature.data is None: continue + if mm_feature.identifier in self.encoder_cache.encoder_outputs: + continue mm_hashes.append(mm_feature.identifier) mm_kwargs.append((mm_feature.modality, mm_feature.data)) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 92b5790ce2..0a1b726c0a 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -63,6 +63,7 @@ from vllm.v1.outputs import ( DraftTokenIds, ModelRunnerOutput, RoutedExpertsTensors, + make_empty_encoder_model_runner_output, ) from vllm.v1.worker.block_table import get_block_table_width from vllm.v1.worker.cp_utils import check_attention_cp_compatibility @@ -86,6 +87,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import ( get_uniform_token_count, ) from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp +from vllm.v1.worker.gpu.ec_connector import get_ec_connector from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after from vllm.v1.worker.gpu.input_batch import ( InputBatch, @@ -147,6 +149,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.device = device self.dtype = self.model_config.dtype + self.is_encoder_only = vllm_config.is_encoder_only self.kv_cache_dtype = self.dtype if self.cache_config.cache_dtype != "auto": # Quantized KV cache. @@ -205,6 +208,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.encoder_cache = None if self.supports_mm_inputs and self.is_first_pp_rank: self.encoder_cache = EncoderCache() + self.ec_connector = get_ec_connector(vllm_config, self.encoder_cache) # Speculative decoding. self.speculator = None @@ -448,6 +452,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): return torch.cuda.current_stream(self.device) def get_kv_cache_spec(self): + if self.is_encoder_only: + return {} return get_kv_cache_spec(self.vllm_config) def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: @@ -580,6 +586,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): is_profile: bool = False, **kwargs, ) -> tuple[torch.Tensor | None, torch.Tensor | None]: + if self.is_encoder_only: + empty = torch.empty(0, device=self.device) + return empty, empty if skip_attn and not is_profile: raise ValueError( "skip_attn must only be True for initial memory profiling." @@ -734,6 +743,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): dummy_mm_inputs, mm_budget ) + if self.is_encoder_only: + torch.accelerator.synchronize() + self.reset_encoder_cache() + gc.collect() + return + hidden_states, sample_hidden_states = self._dummy_run( self.max_num_tokens, skip_attn=True, is_profile=True ) @@ -770,6 +785,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): @torch.inference_mode() def capture_model(self) -> int: + if self.is_encoder_only: + return 0 + assert self.cudagraph_manager is not None if not self.cudagraph_manager.needs_capture(): logger.warning( @@ -1345,6 +1363,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_ids = input_batch.input_ids inputs_embeds = None + ec_connector_output = None if self.supports_mm_inputs and self.is_first_pp_rank: # Run MM encoder (if needed) and get multimodal embeddings. # Only first PP rank prepares multimodal embeddings. @@ -1364,12 +1383,20 @@ class GPUModelRunner(LoRAModelRunnerMixin): lora_state=self.lora_state, scheduled_encoder_inputs=scheduled_encoder_inputs, ) - inputs_embeds = self.model_state.get_mm_embeddings( - scheduled_encoder_inputs, input_batch, self.req_states - ) + with self.ec_connector.maybe_get_output( + scheduler_output + ) as ec_connector_output: + inputs_embeds = self.model_state.get_mm_embeddings( + scheduled_encoder_inputs, input_batch, self.req_states + ) if inputs_embeds is not None and not self.model.requires_raw_input_tokens: input_ids = None + if self.is_encoder_only: + output = make_empty_encoder_model_runner_output(scheduler_output) + output.ec_connector_output = ec_connector_output + return output + model_inputs = { "input_ids": input_ids, "positions": input_batch.positions, diff --git a/vllm/v1/worker/gpu/warmup.py b/vllm/v1/worker/gpu/warmup.py index cb39581c20..7576cdb236 100644 --- a/vllm/v1/worker/gpu/warmup.py +++ b/vllm/v1/worker/gpu/warmup.py @@ -162,6 +162,9 @@ def warmup_kernels( We must call the provided worker's execute_model for pipeline parallel coordination. """ + if model_runner.is_encoder_only: + return + num_spec_steps = model_runner.num_speculative_steps decode_query_len = model_runner.decode_query_len # Use decode_query_len + 1 tokens so the prefill batch's per-request query