[BugFix] Fix use of per-request seed with pipeline parallel (#6698)
This commit is contained in:
@@ -213,6 +213,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
"""
|
||||
self.proposer_worker = proposer_worker
|
||||
self.scorer_worker = scorer_worker
|
||||
scorer_runner = getattr(self.scorer_worker, "model_runner", None)
|
||||
self.generators = scorer_runner.get_generators(
|
||||
) if scorer_runner else None
|
||||
self.disable_by_batch_size = disable_by_batch_size or float("inf")
|
||||
self.spec_decode_sampler = spec_decode_sampler
|
||||
self._allow_zero_draft_token_step = allow_zero_draft_token_step
|
||||
@@ -591,20 +594,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
proposal_token_ids = proposals.proposal_token_ids[spec_indices]
|
||||
|
||||
# Sampler arguments
|
||||
sampler_extra_kwargs = {}
|
||||
if isinstance(self.spec_decode_sampler,
|
||||
SpecDecodeStochasticBaseSampler):
|
||||
|
||||
# Get sequence group state
|
||||
generators = []
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
if (seq_group_metadata.state is not None
|
||||
and seq_group_metadata.state.generator is not None):
|
||||
generators.append(seq_group_metadata.state.generator)
|
||||
else:
|
||||
generators.append(None)
|
||||
|
||||
sampler_extra_kwargs["generators"] = generators
|
||||
sampler_extra_kwargs: Dict[str, Any] = {}
|
||||
if self.generators and isinstance(self.spec_decode_sampler,
|
||||
SpecDecodeStochasticBaseSampler):
|
||||
sampler_extra_kwargs["seeded_seqs"] = {
|
||||
idx: self.generators[sgm.request_id]
|
||||
for idx, sgm in enumerate(seq_group_metadata_list)
|
||||
if sgm.sampling_params.seed is not None
|
||||
}
|
||||
|
||||
accepted_token_ids = self.spec_decode_sampler(
|
||||
target_probs=proposal_verifier_probs,
|
||||
|
||||
Reference in New Issue
Block a user