[EC Connector] Added Build Connector Worker Meta for EC Connector (#49585)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: omerpaz95 <73347585+omerpaz95@users.noreply.github.com> Co-authored-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
yewentao256
Wentao Ye
parent
c94cdd0ae0
commit
fa9d67f782
@@ -65,6 +65,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ECConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsTensors,
|
||||
make_empty_encoder_model_runner_output,
|
||||
@@ -1393,6 +1394,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu
|
||||
)
|
||||
|
||||
def _merge_ec_connector_no_forward(
|
||||
self, scheduler_output: SchedulerOutput, output: ModelRunnerOutput
|
||||
) -> ModelRunnerOutput:
|
||||
"""Let the EC connector send/recv on a step with no work to run."""
|
||||
return ModelRunnerOutput.with_ec_conn_output(
|
||||
output,
|
||||
self.ec_connector.no_forward(scheduler_output).ec_connector_output,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
@@ -1414,7 +1424,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if scheduler_output.total_num_scheduled_tokens == 0:
|
||||
# No need to run the model.
|
||||
empty_output = self.kv_connector.no_forward(scheduler_output)
|
||||
return empty_output
|
||||
return self._merge_ec_connector_no_forward(
|
||||
scheduler_output, empty_output
|
||||
)
|
||||
|
||||
# Get batch descriptor and sync across DP ranks.
|
||||
num_reqs = len(scheduler_output.num_scheduled_tokens)
|
||||
@@ -1455,7 +1467,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if batch_desc.num_tokens == 0:
|
||||
# All DP ranks have zero tokens to run.
|
||||
empty_output = self.kv_connector.no_forward(scheduler_output)
|
||||
return empty_output
|
||||
return self._merge_ec_connector_no_forward(scheduler_output, empty_output)
|
||||
|
||||
if not dummy_run:
|
||||
# Common case.
|
||||
@@ -1572,9 +1584,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_ids = None
|
||||
|
||||
if self.is_encoder_only:
|
||||
output = make_empty_encoder_model_runner_output(scheduler_output)
|
||||
output.ec_connector_output = ec_connector_output
|
||||
return output
|
||||
return ModelRunnerOutput.with_ec_conn_output(
|
||||
make_empty_encoder_model_runner_output(scheduler_output),
|
||||
ec_connector_output,
|
||||
)
|
||||
|
||||
model_inputs = {
|
||||
"input_ids": input_ids,
|
||||
@@ -1679,6 +1692,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states=hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
routed_experts=routed_experts,
|
||||
)
|
||||
|
||||
@@ -1702,6 +1716,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
routed_experts = self.execute_model_state.routed_experts
|
||||
self.execute_model_state = None
|
||||
|
||||
@@ -1721,7 +1736,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
# The first PP rank holds the encoder cache, so pass its EC output on.
|
||||
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
|
||||
|
||||
# Last rank: sample tokens
|
||||
hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling(
|
||||
@@ -1838,6 +1855,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
model_runner_output.kv_connector_output = kv_connector_output
|
||||
model_runner_output.ec_connector_output = ec_connector_output
|
||||
|
||||
return async_output
|
||||
|
||||
@@ -1854,6 +1872,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch = self.execute_model_state.input_batch
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
self.execute_model_state = None
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
@@ -1861,7 +1880,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
|
||||
|
||||
assert self.pooling_runner is not None
|
||||
pooler_output, finished_mask = self.pooling_runner.pool(
|
||||
@@ -1873,6 +1893,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_ids=input_batch.req_ids,
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
|
||||
kv_connector_output=kv_connector_output,
|
||||
ec_connector_output=ec_connector_output,
|
||||
)
|
||||
async_output = AsyncPoolingOutput(
|
||||
model_runner_output=model_runner_output,
|
||||
@@ -1962,6 +1983,7 @@ class ExecuteModelState(NamedTuple):
|
||||
hidden_states: torch.Tensor | None
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
routed_experts: RoutedExpertsTensors | None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user