[Model] MLPSpeculator speculative decoding support (#4947)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Davis Wertheimer <Davis.Wertheimer@ibm.com>
This commit is contained in:
co-authored by
Thomas Parnell
Nick Hill
Davis Wertheimer
parent
6c5b7af152
commit
b12518d3cf
+39
-15
@@ -230,7 +230,8 @@ class ModelConfig:
|
||||
self,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> None:
|
||||
total_num_attention_heads = self.hf_text_config.num_attention_heads
|
||||
total_num_attention_heads = getattr(self.hf_text_config,
|
||||
"num_attention_heads", 0)
|
||||
tensor_parallel_size = parallel_config.tensor_parallel_size
|
||||
if total_num_attention_heads % tensor_parallel_size != 0:
|
||||
raise ValueError(
|
||||
@@ -238,7 +239,8 @@ class ModelConfig:
|
||||
" must be divisible by tensor parallel size "
|
||||
f"({tensor_parallel_size}).")
|
||||
|
||||
total_num_hidden_layers = self.hf_text_config.num_hidden_layers
|
||||
total_num_hidden_layers = getattr(self.hf_text_config,
|
||||
"num_hidden_layers", 0)
|
||||
pipeline_parallel_size = parallel_config.pipeline_parallel_size
|
||||
if total_num_hidden_layers % pipeline_parallel_size != 0:
|
||||
raise ValueError(
|
||||
@@ -341,8 +343,8 @@ class ModelConfig:
|
||||
|
||||
def get_num_attention_heads(self,
|
||||
parallel_config: "ParallelConfig") -> int:
|
||||
return self.hf_text_config.num_attention_heads // \
|
||||
parallel_config.tensor_parallel_size
|
||||
num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
|
||||
return num_heads // parallel_config.tensor_parallel_size
|
||||
|
||||
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
|
||||
total_num_hidden_layers = self.hf_text_config.num_hidden_layers
|
||||
@@ -818,7 +820,8 @@ class SpeculativeConfig:
|
||||
speculative_model (Optional[str]): The name of the speculative
|
||||
model, if provided.
|
||||
num_speculative_tokens (Optional[int]): The number of speculative
|
||||
tokens, if provided.
|
||||
tokens, if provided. Will default to the number in the draft
|
||||
model config if present, otherwise is required.
|
||||
speculative_max_model_len (Optional[int]): The maximum model len of
|
||||
the speculative model. Used when testing the ability to skip
|
||||
speculation for some sequences.
|
||||
@@ -841,24 +844,18 @@ class SpeculativeConfig:
|
||||
the necessary conditions are met, else None.
|
||||
"""
|
||||
|
||||
if speculative_model is None and num_speculative_tokens is None:
|
||||
if speculative_model is None:
|
||||
if num_speculative_tokens is not None:
|
||||
raise ValueError("num_speculative_tokens was provided without "
|
||||
"speculative_model.")
|
||||
return None
|
||||
|
||||
if speculative_model is not None and num_speculative_tokens is None:
|
||||
raise ValueError(
|
||||
"Expected both speculative_model and "
|
||||
"num_speculative_tokens to be provided, but found "
|
||||
f"{speculative_model=} and {num_speculative_tokens=}.")
|
||||
|
||||
if (speculative_disable_by_batch_size is not None
|
||||
and speculative_disable_by_batch_size < 2):
|
||||
raise ValueError("Expect the batch size threshold of disabling "
|
||||
"speculative decoding is > 1, but got "
|
||||
f"{speculative_disable_by_batch_size=}")
|
||||
|
||||
assert (speculative_model is not None
|
||||
and num_speculative_tokens is not None)
|
||||
|
||||
if enable_chunked_prefill:
|
||||
raise ValueError(
|
||||
"Speculative decoding and chunked prefill are "
|
||||
@@ -912,6 +909,27 @@ class SpeculativeConfig:
|
||||
max_logprobs=target_model_config.max_logprobs,
|
||||
)
|
||||
|
||||
if (draft_model_config.hf_config.model_type == "mlp_speculator"
|
||||
and target_parallel_config.world_size != 1):
|
||||
# MLPSpeculator TP support will be added very soon
|
||||
raise ValueError(
|
||||
"Speculative decoding with mlp_speculator models does not "
|
||||
"yet support distributed inferencing (TP > 1).")
|
||||
|
||||
n_predict = getattr(draft_model_config.hf_config, "n_predict",
|
||||
None)
|
||||
if n_predict is not None:
|
||||
if num_speculative_tokens is None:
|
||||
# Default to max value defined in draft model config.
|
||||
num_speculative_tokens = n_predict
|
||||
elif num_speculative_tokens > n_predict:
|
||||
# Verify provided value doesn't exceed the maximum
|
||||
# supported by the draft model.
|
||||
raise ValueError(
|
||||
"Expected both speculative_model and "
|
||||
"num_speculative_tokens to be provided, but found "
|
||||
f"{speculative_model=} and {num_speculative_tokens=}.")
|
||||
|
||||
draft_model_config.max_model_len = (
|
||||
SpeculativeConfig._maybe_override_draft_max_model_len(
|
||||
speculative_max_model_len,
|
||||
@@ -923,6 +941,12 @@ class SpeculativeConfig:
|
||||
SpeculativeConfig.create_draft_parallel_config(
|
||||
target_parallel_config))
|
||||
|
||||
if num_speculative_tokens is None:
|
||||
raise ValueError(
|
||||
"num_speculative_tokens must be provided with "
|
||||
"speculative_model unless the draft model config contains an "
|
||||
"n_predict parameter.")
|
||||
|
||||
return SpeculativeConfig(
|
||||
draft_model_config,
|
||||
draft_parallel_config,
|
||||
|
||||
Reference in New Issue
Block a user