[Core] Cleanup KVConnector handling with PP + fix MRV2 (#43732)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-05-28 13:12:03 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9090368b65
commit 7e53283b1c
9 changed files with 38 additions and 48 deletions
+8 -8
View File
@@ -1218,9 +1218,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if not self.is_last_pp_rank:
# Non-last PP rank: return IntermediateTensors for sending.
assert output_intermediate_tensors is not None
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
output_intermediate_tensors.kv_connector_output = kv_connector_output
return output_intermediate_tensors
return None
@@ -1249,7 +1246,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
)
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
return None
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
# Last rank: sample tokens
sampler_output, num_sampled, num_rejected = self.sample(
@@ -1367,18 +1367,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids = self.execute_model_state.finished_req_ids
self.execute_model_state = None
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
if not self.is_last_pp_rank:
self.postprocess_pool(input_batch)
return None
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
assert self.pooling_runner is not None
pooler_output, is_valid = self.pooling_runner.pool(
hidden_states, input_batch, self.req_states
)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
# Build the model runner output.
model_runner_output = ModelRunnerOutput(
req_ids=input_batch.req_ids,