[EC Connector] Added Build Connector Worker Meta for EC Connector (#49585)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Signed-off-by: omerpaz95 <73347585+omerpaz95@users.noreply.github.com>
Co-authored-by: yewentao256 <zhyanwentao@126.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
omerpaz95
2026-08-15 15:40:38 -07:00
committed by GitHub
co-authored by yewentao256 Wentao Ye
parent c94cdd0ae0
commit fa9d67f782
16 changed files with 430 additions and 16 deletions
+29 -7
View File
@@ -65,6 +65,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import (
DraftTokenIds,
ECConnectorOutput,
ModelRunnerOutput,
RoutedExpertsTensors,
make_empty_encoder_model_runner_output,
@@ -1393,6 +1394,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu
)
def _merge_ec_connector_no_forward(
self, scheduler_output: SchedulerOutput, output: ModelRunnerOutput
) -> ModelRunnerOutput:
"""Let the EC connector send/recv on a step with no work to run."""
return ModelRunnerOutput.with_ec_conn_output(
output,
self.ec_connector.no_forward(scheduler_output).ec_connector_output,
)
@torch.inference_mode()
def execute_model(
self,
@@ -1414,7 +1424,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if scheduler_output.total_num_scheduled_tokens == 0:
# No need to run the model.
empty_output = self.kv_connector.no_forward(scheduler_output)
return empty_output
return self._merge_ec_connector_no_forward(
scheduler_output, empty_output
)
# Get batch descriptor and sync across DP ranks.
num_reqs = len(scheduler_output.num_scheduled_tokens)
@@ -1455,7 +1467,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if batch_desc.num_tokens == 0:
# All DP ranks have zero tokens to run.
empty_output = self.kv_connector.no_forward(scheduler_output)
return empty_output
return self._merge_ec_connector_no_forward(scheduler_output, empty_output)
if not dummy_run:
# Common case.
@@ -1572,9 +1584,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_ids = None
if self.is_encoder_only:
output = make_empty_encoder_model_runner_output(scheduler_output)
output.ec_connector_output = ec_connector_output
return output
return ModelRunnerOutput.with_ec_conn_output(
make_empty_encoder_model_runner_output(scheduler_output),
ec_connector_output,
)
model_inputs = {
"input_ids": input_ids,
@@ -1679,6 +1692,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states=hidden_states,
aux_hidden_states=aux_hidden_states,
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
routed_experts=routed_experts,
)
@@ -1702,6 +1716,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
routed_experts = self.execute_model_state.routed_experts
self.execute_model_state = None
@@ -1721,7 +1736,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
# The first PP rank holds the encoder cache, so pass its EC output on.
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
# Last rank: sample tokens
hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling(
@@ -1838,6 +1855,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
model_runner_output.kv_connector_output = kv_connector_output
model_runner_output.ec_connector_output = ec_connector_output
return async_output
@@ -1854,6 +1872,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch = self.execute_model_state.input_batch
hidden_states = self.execute_model_state.hidden_states
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
self.execute_model_state = None
# Post-step KV connector related operations.
@@ -1861,7 +1880,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if not self.is_last_pp_rank:
self.postprocess_num_computed_tokens(input_batch)
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
assert self.pooling_runner is not None
pooler_output, finished_mask = self.pooling_runner.pool(
@@ -1873,6 +1893,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_ids=input_batch.req_ids,
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
kv_connector_output=kv_connector_output,
ec_connector_output=ec_connector_output,
)
async_output = AsyncPoolingOutput(
model_runner_output=model_runner_output,
@@ -1962,6 +1983,7 @@ class ExecuteModelState(NamedTuple):
hidden_states: torch.Tensor | None
aux_hidden_states: list[torch.Tensor] | None
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
routed_experts: RoutedExpertsTensors | None