[Model Runner V2] Add full cuda graph support for eagle prefill (#37588)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -464,7 +464,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
num_tokens_across_dp = self.execute_model_state.num_tokens_across_dp
|
||||
self.execute_model_state = None
|
||||
|
||||
# dummy run the eagle speculator's propose to ensure DP/EP sync.
|
||||
@@ -496,7 +495,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
next_prefill_tokens=self.req_states.next_prefill_tokens,
|
||||
temperature=self.sampler.sampling_states.temperature.gpu,
|
||||
seeds=self.sampler.sampling_states.seeds.gpu,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
mm_inputs=mm_inputs,
|
||||
@@ -1110,7 +1108,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states=hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
kv_connector_output=kv_connector_output,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -1135,7 +1132,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
kv_connector_output = self.execute_model_state.kv_connector_output
|
||||
num_tokens_across_dp = self.execute_model_state.num_tokens_across_dp
|
||||
self.execute_model_state = None
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -1228,7 +1224,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.next_prefill_tokens,
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
@@ -1336,4 +1331,3 @@ class ExecuteModelState(NamedTuple):
|
||||
hidden_states: torch.Tensor | None
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
kv_connector_output: KVConnectorOutput | None
|
||||
num_tokens_across_dp: torch.Tensor | None
|
||||
|
||||
Reference in New Issue
Block a user