[Speculative decoding 3/9] Worker which speculates, scores, and applies rejection sampling (#3103)
This commit is contained in:
@@ -97,8 +97,6 @@ class ModelRunner:
|
||||
f"Loading model weights took {self.model_memory_usage / float(2**30):.4f} GB"
|
||||
)
|
||||
|
||||
vocab_size = self.model.config.vocab_size
|
||||
|
||||
if self.lora_config:
|
||||
assert hasattr(
|
||||
self.model, "supported_lora_modules"
|
||||
@@ -111,7 +109,7 @@ class ModelRunner:
|
||||
self.lora_manager = LRUCacheWorkerLoRAManager(
|
||||
self.scheduler_config.max_num_seqs,
|
||||
self.scheduler_config.max_num_batched_tokens +
|
||||
self.scheduler_config.max_paddings, vocab_size,
|
||||
self.scheduler_config.max_paddings, self.vocab_size,
|
||||
self.lora_config, self.device, self.model.embedding_modules,
|
||||
self.model.embedding_padding_modules)
|
||||
self.model = self.lora_manager.create_lora_manager(self.model)
|
||||
@@ -607,8 +605,7 @@ class ModelRunner:
|
||||
@torch.inference_mode()
|
||||
def profile_run(self) -> None:
|
||||
# Enable top-k sampling to reflect the accurate memory usage.
|
||||
vocab_size = self.model_config.get_vocab_size()
|
||||
sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
|
||||
sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
|
||||
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
max_num_seqs = self.scheduler_config.max_num_seqs
|
||||
|
||||
@@ -774,6 +771,10 @@ class ModelRunner:
|
||||
self.graph_runners.clear()
|
||||
self.cupy_nccl_backend = None
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
return self.model_config.get_vocab_size()
|
||||
|
||||
|
||||
class CUDAGraphRunner:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user