[Chore] Remove Sampler from Model Code (#17084)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
@@ -295,7 +295,7 @@ class TP1DraftModelRunner(ModelRunnerWrapperBase):
|
||||
if not self.is_driver_worker:
|
||||
return []
|
||||
# Sample the next token.
|
||||
output = self.model.sample(
|
||||
output = self.model_runner.sampler(
|
||||
logits=logits,
|
||||
sampling_metadata=model_input.sampling_metadata,
|
||||
)
|
||||
|
||||
@@ -50,11 +50,10 @@ class MultiStepWorker(ProposerWorkerBase, DelegateWorkerBase):
|
||||
|
||||
def set_include_gpu_probs_tensor(self) -> None:
|
||||
# Need include_gpu_probs_tensor for MultiStepWorker
|
||||
self.model_runner.model.sampler.include_gpu_probs_tensor = True
|
||||
self.model_runner.sampler.include_gpu_probs_tensor = True
|
||||
|
||||
def set_should_modify_greedy_probs_inplace(self) -> None:
|
||||
self.model_runner.model.sampler.should_modify_greedy_probs_inplace = (
|
||||
True)
|
||||
self.model_runner.sampler.should_modify_greedy_probs_inplace = True
|
||||
|
||||
@torch.inference_mode()
|
||||
def sampler_output(
|
||||
|
||||
@@ -410,9 +410,9 @@ class SpecDecodeWorker(LoRANotSupportedWorkerBase):
|
||||
NOTE(cade): This will require a special check if the proposer worker
|
||||
does not have a sampler (e.g. ngram speculation).
|
||||
"""
|
||||
(self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
|
||||
(self.scorer_worker.model_runner.sampler.include_gpu_probs_tensor
|
||||
) = True
|
||||
(self.scorer_worker.model_runner.model.sampler.
|
||||
(self.scorer_worker.model_runner.sampler.
|
||||
should_modify_greedy_probs_inplace) = True
|
||||
self.proposer_worker.set_include_gpu_probs_tensor()
|
||||
self.proposer_worker.set_should_modify_greedy_probs_inplace()
|
||||
|
||||
Reference in New Issue
Block a user