[Quality] Add code formatter and linter (#326)
This commit is contained in:
@@ -93,8 +93,8 @@ class CacheEngine:
|
||||
if not pin_memory:
|
||||
# Pinning memory in WSL is not supported.
|
||||
# https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
|
||||
logger.warn("Using 'pin_memory=False' as WSL is detected. "
|
||||
"This may slow down the performance.")
|
||||
logger.warning("Using 'pin_memory=False' as WSL is detected. "
|
||||
"This may slow down the performance.")
|
||||
for _ in range(self.num_layers):
|
||||
key_blocks = torch.empty(
|
||||
size=(self.num_cpu_blocks, *key_block_shape),
|
||||
@@ -120,11 +120,10 @@ class CacheEngine:
|
||||
src_key_cache, src_value_cache = src[i]
|
||||
dst_key_cache, dst_value_cache = dst[i]
|
||||
# Copy the key blocks.
|
||||
cache_ops.swap_blocks(
|
||||
src_key_cache, dst_key_cache, src_to_dst)
|
||||
cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
|
||||
# Copy the value blocks.
|
||||
cache_ops.swap_blocks(
|
||||
src_value_cache, dst_value_cache, src_to_dst)
|
||||
cache_ops.swap_blocks(src_value_cache, dst_value_cache,
|
||||
src_to_dst)
|
||||
event = self.events[i]
|
||||
event.record(stream=self.cache_stream)
|
||||
|
||||
|
||||
+13
-10
@@ -73,8 +73,8 @@ class Worker:
|
||||
# number of tokens equal to max_num_batched_tokens.
|
||||
|
||||
# Enable top-k sampling to reflect the accurate memory usage.
|
||||
sampling_params = SamplingParams(top_p=0.99,
|
||||
top_k=self.model.config.vocab_size - 1)
|
||||
vocab_size = self.model.config.vocab_size
|
||||
sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
|
||||
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
max_num_seqs = self.scheduler_config.max_num_seqs
|
||||
seqs = []
|
||||
@@ -91,7 +91,8 @@ class Worker:
|
||||
)
|
||||
seqs.append(seq)
|
||||
|
||||
input_tokens, input_positions, input_metadata = self._prepare_inputs(seqs)
|
||||
input_tokens, input_positions, input_metadata = self._prepare_inputs(
|
||||
seqs)
|
||||
|
||||
# Execute the model.
|
||||
num_layers = self.model_config.get_num_layers(self.parallel_config)
|
||||
@@ -110,8 +111,9 @@ class Worker:
|
||||
total_gpu_memory = get_gpu_memory()
|
||||
cache_block_size = CacheEngine.get_cache_block_size(
|
||||
block_size, self.model_config, self.parallel_config)
|
||||
num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
|
||||
- peak_memory) // cache_block_size)
|
||||
num_gpu_blocks = int(
|
||||
(total_gpu_memory * gpu_memory_utilization - peak_memory) //
|
||||
cache_block_size)
|
||||
num_cpu_blocks = int(cpu_swap_space // cache_block_size)
|
||||
num_gpu_blocks = max(num_gpu_blocks, 0)
|
||||
num_cpu_blocks = max(num_cpu_blocks, 0)
|
||||
@@ -125,8 +127,8 @@ class Worker:
|
||||
def init_cache_engine(self, cache_config: CacheConfig) -> None:
|
||||
self.cache_config = cache_config
|
||||
self.block_size = cache_config.block_size
|
||||
self.cache_engine = CacheEngine(
|
||||
self.cache_config, self.model_config, self.parallel_config)
|
||||
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
|
||||
self.parallel_config)
|
||||
self.cache_events = self.cache_engine.events
|
||||
self.gpu_cache = self.cache_engine.gpu_cache
|
||||
|
||||
@@ -202,8 +204,8 @@ class Worker:
|
||||
generation_block_tables.append(block_table)
|
||||
|
||||
max_context_len = max(max_context_len, context_len)
|
||||
max_num_blocks_per_seq = max(
|
||||
max_num_blocks_per_seq, len(block_table))
|
||||
max_num_blocks_per_seq = max(max_num_blocks_per_seq,
|
||||
len(block_table))
|
||||
context_lens.append(context_len)
|
||||
|
||||
block_number = block_table[position // self.block_size]
|
||||
@@ -223,7 +225,8 @@ class Worker:
|
||||
context_lens_tensor = torch.cuda.IntTensor(context_lens)
|
||||
padded_block_tables = [
|
||||
_pad_to_max(block_table, max_num_blocks_per_seq)
|
||||
for block_table in generation_block_tables]
|
||||
for block_table in generation_block_tables
|
||||
]
|
||||
block_tables_tensor = torch.cuda.IntTensor(padded_block_tables)
|
||||
|
||||
seq_data: Dict[int, SequenceData] = {}
|
||||
|
||||
Reference in New Issue
Block a user