[Mistral] Mistral-7B-v0.1 support (#1196)

Co-authored-by: timlacroix <t@mistral.ai>
This commit is contained in:
Chris Bamford
2023-09-28 10:41:03 -07:00
committed by GitHub
co-authored by timlacroix
parent 7bedab5748
commit bb1ba58f06
13 changed files with 571 additions and 25 deletions
+17 -3
View File
@@ -137,8 +137,10 @@ class Worker:
self.cache_config = cache_config
self.block_size = cache_config.block_size
_check_if_can_support_max_seq_len(self.scheduler_config.max_model_len,
self.block_size)
max_seq_len = min(self.scheduler_config.max_model_len,
cache_config.sliding_window or float("inf"))
_check_if_can_support_max_seq_len(max_seq_len, self.block_size)
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
self.parallel_config)
@@ -191,6 +193,9 @@ class Worker:
slot = block_number * self.block_size + block_offset
slot_mapping.append(slot)
sliding_window = getattr(self.model_config.hf_config, "sliding_window",
float("inf"))
# Add generation tokens.
max_context_len = 0
max_num_blocks_per_seq = 0
@@ -211,10 +216,11 @@ class Worker:
context_len = seq_data.get_len()
position = context_len - 1
if sliding_window:
context_len = min(context_len, sliding_window)
input_positions.append(position)
block_table = seq_group_metadata.block_tables[seq_id]
generation_block_tables.append(block_table)
max_context_len = max(max_context_len, context_len)
max_num_blocks_per_seq = max(max_num_blocks_per_seq,
@@ -226,6 +232,13 @@ class Worker:
slot = block_number * self.block_size + block_offset
slot_mapping.append(slot)
if sliding_window:
assert self.cache_config is not None
sliding_window_blocks = (sliding_window //
self.cache_config.block_size)
block_table = block_table[-sliding_window_blocks:]
generation_block_tables.append(block_table)
# Optimization: Pad the input length to be a multiple of 8.
# This is required for utilizing the Tensor Cores in NVIDIA GPUs.
input_tokens = _pad_to_alignment(input_tokens, multiple_of=8)
@@ -264,6 +277,7 @@ class Worker:
context_lens=context_lens_tensor,
max_context_len=max_context_len,
block_tables=block_tables_tensor,
sliding_window=sliding_window,
)
return tokens_tensor, positions_tensor, input_metadata