[Speculative decoding 3/9] Worker which speculates, scores, and applies rejection sampling (#3103)

This commit is contained in:
Cade Daniel
2024-03-08 23:32:46 -08:00
committed by GitHub
parent f48c6791b7
commit 8437bae6ef
21 changed files with 2786 additions and 215 deletions
+18 -2
View File
@@ -130,8 +130,8 @@ class Worker:
# GPU did not change their memory usage during the profiling.
peak_memory = self.init_gpu_memory - free_gpu_memory
cache_block_size = CacheEngine.get_cache_block_size(
block_size, cache_dtype, self.model_config, self.parallel_config)
cache_block_size = self.get_cache_block_size_bytes(
block_size, cache_dtype)
num_gpu_blocks = int(
(total_gpu_memory * gpu_memory_utilization - peak_memory) //
cache_block_size)
@@ -232,6 +232,22 @@ class Worker:
def list_loras(self) -> Set[int]:
return self.model_runner.list_loras()
@property
def max_model_len(self) -> int:
return self.model_config.max_model_len
@property
def vocab_size(self) -> int:
return self.model_runner.vocab_size
def get_cache_block_size_bytes(self, block_size: int,
cache_dtype: str) -> int:
"""Get the size of the KV cache block size in bytes.
"""
return CacheEngine.get_cache_block_size(block_size, cache_dtype,
self.model_config,
self.parallel_config)
def init_distributed_environment(
parallel_config: ParallelConfig,