Add distributed model executor abstraction (#3191)

This commit is contained in:
Zhuohan Li
2024-03-11 11:03:45 -07:00
committed by GitHub
parent 657061fdce
commit 4c922709b6
13 changed files with 817 additions and 508 deletions
+13
View File
@@ -0,0 +1,13 @@
def check_block_size_valid(num_gpu_blocks, block_size, max_model_len) -> None:
if num_gpu_blocks <= 0:
raise ValueError("No available memory for the cache blocks. "
"Try increasing `gpu_memory_utilization` when "
"initializing the engine.")
max_seq_len = block_size * num_gpu_blocks
if max_model_len > max_seq_len:
raise ValueError(
f"The model's max seq len ({max_model_len}) "
"is larger than the maximum number of tokens that can be "
f"stored in KV cache ({max_seq_len}). Try increasing "
"`gpu_memory_utilization` or decreasing `max_model_len` when "
"initializing the engine.")