[Model Runner v2] E/P/D disaggregation support (#38390)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Wentao Ye
2026-08-04 15:52:23 +00:00
committed by GitHub
parent 7743486190
commit 4f819f801b
7 changed files with 145 additions and 8 deletions
@@ -29,7 +29,7 @@ if [[ -z "${DEVICE_AFFINITY_ENV:-}" ]]; then
fi
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-12000}" # wait_for_server timeout
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-300}" # wait_for_server timeout
NUM_PROMPTS="${NUM_PROMPTS:-100}" # number of prompts to send in benchmark
+18 -4
View File
@@ -536,6 +536,24 @@ class VllmConfig:
]
return hash_str
@property
def is_ec_producer_only(self) -> bool:
ec_config = self.ec_transfer_config
return (
ec_config is not None
and ec_config.is_ec_producer
and not ec_config.is_ec_consumer
)
@property
def is_encoder_only(self) -> bool:
mm_config = (
self.model_config.multimodal_config
if self.model_config is not None
else None
)
return self.is_ec_producer_only or bool(mm_config and mm_config.mm_encoder_only)
@property
def max_concurrent_batches(self) -> int:
# PP requires PP-size concurrent batches to fill the pipeline.
@@ -2246,10 +2264,6 @@ class VllmConfig:
# Will be added by https://github.com/vllm-project/vllm/pull/35045
unsupported.append("KV sharing fast prefill")
if self.ec_transfer_config is not None:
# Will be added by https://github.com/vllm-project/vllm/pull/38390
unsupported.append("EC transfer")
return unsupported
def _validate_v2_model_runner(self) -> None:
+6
View File
@@ -120,6 +120,8 @@ class BlockTables:
self.num_blocks.np[i, req_index] = start + len(block_ids)
def apply_staged_writes(self) -> None:
if self.num_kv_cache_groups == 0:
return
if self.num_kv_cache_groups == 1:
# Single group: write directly, skipping the per-write group lookup.
self.block_tables[0].apply_write()
@@ -138,6 +140,8 @@ class BlockTables:
out: tuple[torch.Tensor, ...] | None = None,
out_ptrs: torch.Tensor | None = None,
) -> tuple[torch.Tensor, ...]:
if self.num_kv_cache_groups == 0:
return ()
if out is None:
out = tuple(self.input_block_tables)
out_ptrs = self.input_block_table_ptrs
@@ -179,6 +183,8 @@ class BlockTables:
num_tokens_padded: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
if self.num_kv_cache_groups == 0:
return (self.slot_mappings if out is None else out)[:, :num_tokens_padded]
num_reqs = idx_mapping.shape[0]
num_groups = self.num_kv_cache_groups
slot_mappings = self.slot_mappings if out is None else out
+85
View File
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Generator
from contextlib import contextmanager
from typing import TYPE_CHECKING
import torch
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase
from vllm.v1.outputs import ECConnectorOutput
if TYPE_CHECKING:
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
class ECConnector:
"""EC connector interface used by the V2 GPU model runner."""
@contextmanager
def maybe_get_output(
self, scheduler_output: "SchedulerOutput"
) -> Generator[ECConnectorOutput | None, None, None]:
yield None
class ActiveECConnector(ECConnector):
def __init__(
self,
vllm_config: VllmConfig,
encoder_cache: dict[str, torch.Tensor],
) -> None:
self.encoder_cache = encoder_cache
self.save_new_caches = vllm_config.is_ec_producer_only
self.ec_connector = get_ec_transfer()
assert isinstance(self.ec_connector, ECConnectorBase)
@contextmanager
def maybe_get_output(
self, scheduler_output: "SchedulerOutput"
) -> Generator[ECConnectorOutput | None, None, None]:
if scheduler_output.ec_connector_metadata is None:
yield None
return
output = ECConnectorOutput()
ec_connector = self.ec_connector
assert scheduler_output.ec_connector_metadata is not None
ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata)
if ec_connector.is_consumer:
ec_connector.start_load_caches(self.encoder_cache)
cached_hashes = set(self.encoder_cache) if self.save_new_caches else None
try:
yield output
if cached_hashes is not None:
for mm_hash in self.encoder_cache.keys() - cached_hashes:
ec_connector.save_caches(
encoder_cache=self.encoder_cache, mm_hash=mm_hash
)
finally:
output.finished_sending, output.finished_recving = (
ec_connector.get_finished(scheduler_output.finished_req_ids)
)
ec_connector.clear_connector_metadata()
NO_OP_EC_CONNECTOR = ECConnector()
def get_ec_connector(
vllm_config: VllmConfig,
encoder_cache: "EncoderCache | None",
) -> ECConnector:
if (
not has_ec_transfer()
or vllm_config.model_config.is_encoder_decoder
or encoder_cache is None
):
return NO_OP_EC_CONNECTOR
return ActiveECConnector(vllm_config, encoder_cache.encoder_outputs)
+2
View File
@@ -51,6 +51,8 @@ class EncoderRunner:
mm_feature = mm_features[mm_input_id]
if mm_feature.data is None:
continue
if mm_feature.identifier in self.encoder_cache.encoder_outputs:
continue
mm_hashes.append(mm_feature.identifier)
mm_kwargs.append((mm_feature.modality, mm_feature.data))
+30 -3
View File
@@ -63,6 +63,7 @@ from vllm.v1.outputs import (
DraftTokenIds,
ModelRunnerOutput,
RoutedExpertsTensors,
make_empty_encoder_model_runner_output,
)
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
@@ -86,6 +87,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
get_uniform_token_count,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
from vllm.v1.worker.gpu.input_batch import (
InputBatch,
@@ -147,6 +149,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.device = device
self.dtype = self.model_config.dtype
self.is_encoder_only = vllm_config.is_encoder_only
self.kv_cache_dtype = self.dtype
if self.cache_config.cache_dtype != "auto":
# Quantized KV cache.
@@ -205,6 +208,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.encoder_cache = None
if self.supports_mm_inputs and self.is_first_pp_rank:
self.encoder_cache = EncoderCache()
self.ec_connector = get_ec_connector(vllm_config, self.encoder_cache)
# Speculative decoding.
self.speculator = None
@@ -448,6 +452,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return torch.cuda.current_stream(self.device)
def get_kv_cache_spec(self):
if self.is_encoder_only:
return {}
return get_kv_cache_spec(self.vllm_config)
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
@@ -580,6 +586,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
is_profile: bool = False,
**kwargs,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
if self.is_encoder_only:
empty = torch.empty(0, device=self.device)
return empty, empty
if skip_attn and not is_profile:
raise ValueError(
"skip_attn must only be True for initial memory profiling."
@@ -734,6 +743,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dummy_mm_inputs, mm_budget
)
if self.is_encoder_only:
torch.accelerator.synchronize()
self.reset_encoder_cache()
gc.collect()
return
hidden_states, sample_hidden_states = self._dummy_run(
self.max_num_tokens, skip_attn=True, is_profile=True
)
@@ -770,6 +785,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
@torch.inference_mode()
def capture_model(self) -> int:
if self.is_encoder_only:
return 0
assert self.cudagraph_manager is not None
if not self.cudagraph_manager.needs_capture():
logger.warning(
@@ -1345,6 +1363,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_ids = input_batch.input_ids
inputs_embeds = None
ec_connector_output = None
if self.supports_mm_inputs and self.is_first_pp_rank:
# Run MM encoder (if needed) and get multimodal embeddings.
# Only first PP rank prepares multimodal embeddings.
@@ -1364,12 +1383,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
lora_state=self.lora_state,
scheduled_encoder_inputs=scheduled_encoder_inputs,
)
inputs_embeds = self.model_state.get_mm_embeddings(
scheduled_encoder_inputs, input_batch, self.req_states
)
with self.ec_connector.maybe_get_output(
scheduler_output
) as ec_connector_output:
inputs_embeds = self.model_state.get_mm_embeddings(
scheduled_encoder_inputs, input_batch, self.req_states
)
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
input_ids = None
if self.is_encoder_only:
output = make_empty_encoder_model_runner_output(scheduler_output)
output.ec_connector_output = ec_connector_output
return output
model_inputs = {
"input_ids": input_ids,
"positions": input_batch.positions,
+3
View File
@@ -162,6 +162,9 @@ def warmup_kernels(
We must call the provided worker's execute_model for pipeline parallel
coordination.
"""
if model_runner.is_encoder_only:
return
num_spec_steps = model_runner.num_speculative_steps
decode_query_len = model_runner.decode_query_len
# Use decode_query_len + 1 tokens so the prefill batch's per-request query