[Core] Cleanup KVConnector handling with PP + fix MRV2 (#43732)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9090368b65
commit
7e53283b1c
@@ -1218,9 +1218,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
# Non-last PP rank: return IntermediateTensors for sending.
|
||||
assert output_intermediate_tensors is not None
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
output_intermediate_tensors.kv_connector_output = kv_connector_output
|
||||
return output_intermediate_tensors
|
||||
return None
|
||||
|
||||
@@ -1249,7 +1246,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
|
||||
)
|
||||
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
|
||||
return None
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
|
||||
# Last rank: sample tokens
|
||||
sampler_output, num_sampled, num_rejected = self.sample(
|
||||
@@ -1367,18 +1367,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
self.execute_model_state = None
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
self.postprocess_pool(input_batch)
|
||||
return None
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
|
||||
assert self.pooling_runner is not None
|
||||
pooler_output, is_valid = self.pooling_runner.pool(
|
||||
hidden_states, input_batch, self.req_states
|
||||
)
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
|
||||
# Build the model runner output.
|
||||
model_runner_output = ModelRunnerOutput(
|
||||
req_ids=input_batch.req_ids,
|
||||
|
||||
Reference in New Issue
Block a user