[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -2057,6 +2057,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
num_spec_tokens_to_schedule=scheduler_output.num_spec_tokens_to_schedule,
|
||||
)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -2086,6 +2087,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||
num_spec_tokens = self.execute_model_state.num_spec_tokens_to_schedule
|
||||
self.execute_model_state = None
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -2237,9 +2239,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.next_prefill_tokens,
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
dp_sync=dp_sync,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
if num_spec_tokens < self.num_speculative_steps:
|
||||
draft_tokens[:, num_spec_tokens:] = -1
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
if self.adaptive_verification is not None:
|
||||
self.adaptive_verification.record_confidences(
|
||||
@@ -2251,7 +2256,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# not have a speculator (i.e. self.speculator is None)
|
||||
self.draft_tokens_handler.set_draft_tokens(
|
||||
input_batch,
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping],
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping, :num_spec_tokens],
|
||||
)
|
||||
if self.pp_handler is not None:
|
||||
self.pp_handler.broadcast_drafts(
|
||||
@@ -2396,6 +2401,7 @@ class ExecuteModelState(NamedTuple):
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
num_spec_tokens_to_schedule: int
|
||||
|
||||
|
||||
class BatchReqState(NamedTuple):
|
||||
|
||||
Reference in New Issue
Block a user