[Mistral] Mistral-7B-v0.1 support (#1196)
Co-authored-by: timlacroix <t@mistral.ai>
This commit is contained in:
co-authored by
timlacroix
parent
7bedab5748
commit
bb1ba58f06
+17
-3
@@ -137,8 +137,10 @@ class Worker:
|
||||
self.cache_config = cache_config
|
||||
self.block_size = cache_config.block_size
|
||||
|
||||
_check_if_can_support_max_seq_len(self.scheduler_config.max_model_len,
|
||||
self.block_size)
|
||||
max_seq_len = min(self.scheduler_config.max_model_len,
|
||||
cache_config.sliding_window or float("inf"))
|
||||
|
||||
_check_if_can_support_max_seq_len(max_seq_len, self.block_size)
|
||||
|
||||
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
|
||||
self.parallel_config)
|
||||
@@ -191,6 +193,9 @@ class Worker:
|
||||
slot = block_number * self.block_size + block_offset
|
||||
slot_mapping.append(slot)
|
||||
|
||||
sliding_window = getattr(self.model_config.hf_config, "sliding_window",
|
||||
float("inf"))
|
||||
|
||||
# Add generation tokens.
|
||||
max_context_len = 0
|
||||
max_num_blocks_per_seq = 0
|
||||
@@ -211,10 +216,11 @@ class Worker:
|
||||
|
||||
context_len = seq_data.get_len()
|
||||
position = context_len - 1
|
||||
if sliding_window:
|
||||
context_len = min(context_len, sliding_window)
|
||||
input_positions.append(position)
|
||||
|
||||
block_table = seq_group_metadata.block_tables[seq_id]
|
||||
generation_block_tables.append(block_table)
|
||||
|
||||
max_context_len = max(max_context_len, context_len)
|
||||
max_num_blocks_per_seq = max(max_num_blocks_per_seq,
|
||||
@@ -226,6 +232,13 @@ class Worker:
|
||||
slot = block_number * self.block_size + block_offset
|
||||
slot_mapping.append(slot)
|
||||
|
||||
if sliding_window:
|
||||
assert self.cache_config is not None
|
||||
sliding_window_blocks = (sliding_window //
|
||||
self.cache_config.block_size)
|
||||
block_table = block_table[-sliding_window_blocks:]
|
||||
generation_block_tables.append(block_table)
|
||||
|
||||
# Optimization: Pad the input length to be a multiple of 8.
|
||||
# This is required for utilizing the Tensor Cores in NVIDIA GPUs.
|
||||
input_tokens = _pad_to_alignment(input_tokens, multiple_of=8)
|
||||
@@ -264,6 +277,7 @@ class Worker:
|
||||
context_lens=context_lens_tensor,
|
||||
max_context_len=max_context_len,
|
||||
block_tables=block_tables_tensor,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
return tokens_tensor, positions_tensor, input_metadata
|
||||
|
||||
|
||||
Reference in New Issue
Block a user