[Model Runner v2] E/P/D disaggregation support (#38390)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
@@ -29,7 +29,7 @@ if [[ -z "${DEVICE_AFFINITY_ENV:-}" ]]; then
|
||||
fi
|
||||
|
||||
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache}"
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-12000}" # wait_for_server timeout
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-300}" # wait_for_server timeout
|
||||
|
||||
NUM_PROMPTS="${NUM_PROMPTS:-100}" # number of prompts to send in benchmark
|
||||
|
||||
|
||||
+18
-4
@@ -536,6 +536,24 @@ class VllmConfig:
|
||||
]
|
||||
return hash_str
|
||||
|
||||
@property
|
||||
def is_ec_producer_only(self) -> bool:
|
||||
ec_config = self.ec_transfer_config
|
||||
return (
|
||||
ec_config is not None
|
||||
and ec_config.is_ec_producer
|
||||
and not ec_config.is_ec_consumer
|
||||
)
|
||||
|
||||
@property
|
||||
def is_encoder_only(self) -> bool:
|
||||
mm_config = (
|
||||
self.model_config.multimodal_config
|
||||
if self.model_config is not None
|
||||
else None
|
||||
)
|
||||
return self.is_ec_producer_only or bool(mm_config and mm_config.mm_encoder_only)
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
# PP requires PP-size concurrent batches to fill the pipeline.
|
||||
@@ -2246,10 +2264,6 @@ class VllmConfig:
|
||||
# Will be added by https://github.com/vllm-project/vllm/pull/35045
|
||||
unsupported.append("KV sharing fast prefill")
|
||||
|
||||
if self.ec_transfer_config is not None:
|
||||
# Will be added by https://github.com/vllm-project/vllm/pull/38390
|
||||
unsupported.append("EC transfer")
|
||||
|
||||
return unsupported
|
||||
|
||||
def _validate_v2_model_runner(self) -> None:
|
||||
|
||||
@@ -120,6 +120,8 @@ class BlockTables:
|
||||
self.num_blocks.np[i, req_index] = start + len(block_ids)
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
if self.num_kv_cache_groups == 0:
|
||||
return
|
||||
if self.num_kv_cache_groups == 1:
|
||||
# Single group: write directly, skipping the per-write group lookup.
|
||||
self.block_tables[0].apply_write()
|
||||
@@ -138,6 +140,8 @@ class BlockTables:
|
||||
out: tuple[torch.Tensor, ...] | None = None,
|
||||
out_ptrs: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
if self.num_kv_cache_groups == 0:
|
||||
return ()
|
||||
if out is None:
|
||||
out = tuple(self.input_block_tables)
|
||||
out_ptrs = self.input_block_table_ptrs
|
||||
@@ -179,6 +183,8 @@ class BlockTables:
|
||||
num_tokens_padded: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.num_kv_cache_groups == 0:
|
||||
return (self.slot_mappings if out is None else out)[:, :num_tokens_padded]
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
num_groups = self.num_kv_cache_groups
|
||||
slot_mappings = self.slot_mappings if out is None else out
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
|
||||
|
||||
class ECConnector:
|
||||
"""EC connector interface used by the V2 GPU model runner."""
|
||||
|
||||
@contextmanager
|
||||
def maybe_get_output(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> Generator[ECConnectorOutput | None, None, None]:
|
||||
yield None
|
||||
|
||||
|
||||
class ActiveECConnector(ECConnector):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
) -> None:
|
||||
self.encoder_cache = encoder_cache
|
||||
self.save_new_caches = vllm_config.is_ec_producer_only
|
||||
self.ec_connector = get_ec_transfer()
|
||||
assert isinstance(self.ec_connector, ECConnectorBase)
|
||||
|
||||
@contextmanager
|
||||
def maybe_get_output(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> Generator[ECConnectorOutput | None, None, None]:
|
||||
if scheduler_output.ec_connector_metadata is None:
|
||||
yield None
|
||||
return
|
||||
|
||||
output = ECConnectorOutput()
|
||||
ec_connector = self.ec_connector
|
||||
assert scheduler_output.ec_connector_metadata is not None
|
||||
ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata)
|
||||
|
||||
if ec_connector.is_consumer:
|
||||
ec_connector.start_load_caches(self.encoder_cache)
|
||||
|
||||
cached_hashes = set(self.encoder_cache) if self.save_new_caches else None
|
||||
try:
|
||||
yield output
|
||||
if cached_hashes is not None:
|
||||
for mm_hash in self.encoder_cache.keys() - cached_hashes:
|
||||
ec_connector.save_caches(
|
||||
encoder_cache=self.encoder_cache, mm_hash=mm_hash
|
||||
)
|
||||
finally:
|
||||
output.finished_sending, output.finished_recving = (
|
||||
ec_connector.get_finished(scheduler_output.finished_req_ids)
|
||||
)
|
||||
ec_connector.clear_connector_metadata()
|
||||
|
||||
|
||||
NO_OP_EC_CONNECTOR = ECConnector()
|
||||
|
||||
|
||||
def get_ec_connector(
|
||||
vllm_config: VllmConfig,
|
||||
encoder_cache: "EncoderCache | None",
|
||||
) -> ECConnector:
|
||||
if (
|
||||
not has_ec_transfer()
|
||||
or vllm_config.model_config.is_encoder_decoder
|
||||
or encoder_cache is None
|
||||
):
|
||||
return NO_OP_EC_CONNECTOR
|
||||
|
||||
return ActiveECConnector(vllm_config, encoder_cache.encoder_outputs)
|
||||
@@ -51,6 +51,8 @@ class EncoderRunner:
|
||||
mm_feature = mm_features[mm_input_id]
|
||||
if mm_feature.data is None:
|
||||
continue
|
||||
if mm_feature.identifier in self.encoder_cache.encoder_outputs:
|
||||
continue
|
||||
mm_hashes.append(mm_feature.identifier)
|
||||
mm_kwargs.append((mm_feature.modality, mm_feature.data))
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsTensors,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
@@ -86,6 +87,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
|
||||
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
|
||||
from vllm.v1.worker.gpu.input_batch import (
|
||||
InputBatch,
|
||||
@@ -147,6 +149,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
self.device = device
|
||||
self.dtype = self.model_config.dtype
|
||||
self.is_encoder_only = vllm_config.is_encoder_only
|
||||
self.kv_cache_dtype = self.dtype
|
||||
if self.cache_config.cache_dtype != "auto":
|
||||
# Quantized KV cache.
|
||||
@@ -205,6 +208,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.encoder_cache = None
|
||||
if self.supports_mm_inputs and self.is_first_pp_rank:
|
||||
self.encoder_cache = EncoderCache()
|
||||
self.ec_connector = get_ec_connector(vllm_config, self.encoder_cache)
|
||||
|
||||
# Speculative decoding.
|
||||
self.speculator = None
|
||||
@@ -448,6 +452,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return torch.cuda.current_stream(self.device)
|
||||
|
||||
def get_kv_cache_spec(self):
|
||||
if self.is_encoder_only:
|
||||
return {}
|
||||
return get_kv_cache_spec(self.vllm_config)
|
||||
|
||||
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
@@ -580,6 +586,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
is_profile: bool = False,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
if self.is_encoder_only:
|
||||
empty = torch.empty(0, device=self.device)
|
||||
return empty, empty
|
||||
if skip_attn and not is_profile:
|
||||
raise ValueError(
|
||||
"skip_attn must only be True for initial memory profiling."
|
||||
@@ -734,6 +743,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dummy_mm_inputs, mm_budget
|
||||
)
|
||||
|
||||
if self.is_encoder_only:
|
||||
torch.accelerator.synchronize()
|
||||
self.reset_encoder_cache()
|
||||
gc.collect()
|
||||
return
|
||||
|
||||
hidden_states, sample_hidden_states = self._dummy_run(
|
||||
self.max_num_tokens, skip_attn=True, is_profile=True
|
||||
)
|
||||
@@ -770,6 +785,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture_model(self) -> int:
|
||||
if self.is_encoder_only:
|
||||
return 0
|
||||
|
||||
assert self.cudagraph_manager is not None
|
||||
if not self.cudagraph_manager.needs_capture():
|
||||
logger.warning(
|
||||
@@ -1345,6 +1363,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
input_ids = input_batch.input_ids
|
||||
inputs_embeds = None
|
||||
ec_connector_output = None
|
||||
if self.supports_mm_inputs and self.is_first_pp_rank:
|
||||
# Run MM encoder (if needed) and get multimodal embeddings.
|
||||
# Only first PP rank prepares multimodal embeddings.
|
||||
@@ -1364,12 +1383,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
lora_state=self.lora_state,
|
||||
scheduled_encoder_inputs=scheduled_encoder_inputs,
|
||||
)
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
with self.ec_connector.maybe_get_output(
|
||||
scheduler_output
|
||||
) as ec_connector_output:
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
|
||||
input_ids = None
|
||||
|
||||
if self.is_encoder_only:
|
||||
output = make_empty_encoder_model_runner_output(scheduler_output)
|
||||
output.ec_connector_output = ec_connector_output
|
||||
return output
|
||||
|
||||
model_inputs = {
|
||||
"input_ids": input_ids,
|
||||
"positions": input_batch.positions,
|
||||
|
||||
@@ -162,6 +162,9 @@ def warmup_kernels(
|
||||
We must call the provided worker's execute_model for pipeline parallel
|
||||
coordination.
|
||||
"""
|
||||
if model_runner.is_encoder_only:
|
||||
return
|
||||
|
||||
num_spec_steps = model_runner.num_speculative_steps
|
||||
decode_query_len = model_runner.decode_query_len
|
||||
# Use decode_query_len + 1 tokens so the prefill batch's per-request query
|
||||
|
||||
Reference in New Issue
Block a user