[Speculative decoding 7/9] Speculative decoding end-to-end correctness tests. (#3951)
This commit is contained in:
+65
-2
@@ -655,6 +655,9 @@ class SpeculativeConfig:
|
||||
target_dtype: str,
|
||||
speculative_model: Optional[str],
|
||||
num_speculative_tokens: Optional[int],
|
||||
speculative_max_model_len: Optional[int],
|
||||
enable_chunked_prefill: bool,
|
||||
use_v2_block_manager: bool,
|
||||
) -> Optional["SpeculativeConfig"]:
|
||||
"""Create a SpeculativeConfig if possible, else return None.
|
||||
|
||||
@@ -672,6 +675,15 @@ class SpeculativeConfig:
|
||||
model, if provided.
|
||||
num_speculative_tokens (Optional[int]): The number of speculative
|
||||
tokens, if provided.
|
||||
speculative_max_model_len (Optional[int]): The maximum model len of
|
||||
the speculative model. Used when testing the ability to skip
|
||||
speculation for some sequences.
|
||||
enable_chunked_prefill (bool): Whether vLLM is configured to use
|
||||
chunked prefill or not. Used for raising an error since its not
|
||||
yet compatible with spec decode.
|
||||
use_v2_block_manager (bool): Whether vLLM is configured to use the
|
||||
v2 block manager or not. Used for raising an error since the v2
|
||||
block manager is required with spec decode.
|
||||
|
||||
Returns:
|
||||
Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
|
||||
@@ -690,12 +702,21 @@ class SpeculativeConfig:
|
||||
assert (speculative_model is not None
|
||||
and num_speculative_tokens is not None)
|
||||
|
||||
if enable_chunked_prefill:
|
||||
raise ValueError(
|
||||
"Speculative decoding and chunked prefill are "
|
||||
f"currently mutually exclusive ({enable_chunked_prefill=}).")
|
||||
|
||||
if not use_v2_block_manager:
|
||||
raise ValueError(
|
||||
"Speculative decoding requires usage of the V2 "
|
||||
"block manager. Enable it with --use-v2-block-manager.")
|
||||
|
||||
# TODO: The user should be able to specify revision/quantization/max
|
||||
# model len for the draft model. It is not currently supported.
|
||||
draft_revision = None
|
||||
draft_code_revision = None
|
||||
draft_quantization = None
|
||||
draft_max_model_len = None
|
||||
|
||||
draft_model_config = ModelConfig(
|
||||
model=speculative_model,
|
||||
@@ -707,7 +728,7 @@ class SpeculativeConfig:
|
||||
revision=draft_revision,
|
||||
code_revision=draft_code_revision,
|
||||
tokenizer_revision=target_model_config.tokenizer_revision,
|
||||
max_model_len=draft_max_model_len,
|
||||
max_model_len=None,
|
||||
quantization=draft_quantization,
|
||||
enforce_eager=target_model_config.enforce_eager,
|
||||
max_context_len_to_capture=target_model_config.
|
||||
@@ -715,6 +736,13 @@ class SpeculativeConfig:
|
||||
max_logprobs=target_model_config.max_logprobs,
|
||||
)
|
||||
|
||||
draft_model_config.max_model_len = (
|
||||
SpeculativeConfig._maybe_override_draft_max_model_len(
|
||||
speculative_max_model_len,
|
||||
draft_model_config.max_model_len,
|
||||
target_model_config.max_model_len,
|
||||
))
|
||||
|
||||
draft_parallel_config = (
|
||||
SpeculativeConfig.create_draft_parallel_config(
|
||||
target_parallel_config))
|
||||
@@ -725,6 +753,41 @@ class SpeculativeConfig:
|
||||
num_speculative_tokens,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _maybe_override_draft_max_model_len(
|
||||
speculative_max_model_len: Optional[int],
|
||||
draft_max_model_len: int,
|
||||
target_max_model_len: int,
|
||||
) -> int:
|
||||
"""Determine the max sequence len for the draft model. This is usually
|
||||
the draft_max_model_len, but may be the target_max_model_len if it is
|
||||
less than the draft_max_model_len, or may be speculative_max_model_len
|
||||
if it is specified.
|
||||
|
||||
This is necessary so that sequences do not exceed the capacity of the
|
||||
draft model or the target model.
|
||||
|
||||
speculative_max_model_len is mainly used for testing that sequences can
|
||||
skip speculation.
|
||||
"""
|
||||
|
||||
if speculative_max_model_len is not None:
|
||||
|
||||
if speculative_max_model_len > draft_max_model_len:
|
||||
raise ValueError(f"{speculative_max_model_len=} cannot be "
|
||||
f"larger than {draft_max_model_len=}")
|
||||
|
||||
if speculative_max_model_len > target_max_model_len:
|
||||
raise ValueError(f"{speculative_max_model_len=} cannot be "
|
||||
f"larger than {target_max_model_len=}")
|
||||
|
||||
return speculative_max_model_len
|
||||
|
||||
return min(
|
||||
draft_max_model_len,
|
||||
target_max_model_len,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def create_draft_parallel_config(
|
||||
target_parallel_config: ParallelConfig) -> ParallelConfig:
|
||||
|
||||
Reference in New Issue
Block a user