[Speculative decoding] [Multi-Step] decouple should_modify_greedy_probs_inplace (#6971)
This commit is contained in:
@@ -295,7 +295,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
"""
|
||||
(self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
|
||||
) = True
|
||||
(self.scorer_worker.model_runner.model.sampler.
|
||||
should_modify_greedy_probs_inplace) = True
|
||||
self.proposer_worker.set_include_gpu_probs_tensor()
|
||||
self.proposer_worker.set_should_modify_greedy_probs_inplace()
|
||||
|
||||
def determine_num_available_blocks(self) -> Tuple[int, int]:
|
||||
"""Determine the number of cache blocks to use.
|
||||
|
||||
Reference in New Issue
Block a user