[Bugfix][Core] Fix num_output_placeholders underflow with async scheduling + spec decode (#46066)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
641cb59592
commit
373eb314af
@@ -54,6 +54,7 @@ def create_scheduler(
|
||||
block_size: int = 16,
|
||||
max_model_len: int | None = None,
|
||||
num_speculative_tokens: int | None = None,
|
||||
speculative_method: str | None = None,
|
||||
skip_tokenizer_init: bool = False,
|
||||
async_scheduling: bool = False,
|
||||
pipeline_parallel_size: int = 1,
|
||||
@@ -126,9 +127,14 @@ def create_scheduler(
|
||||
|
||||
speculative_config: SpeculativeConfig | None = None
|
||||
if num_speculative_tokens is not None:
|
||||
speculative_config = SpeculativeConfig(
|
||||
spec_kwargs: dict = dict(
|
||||
model="ngram", num_speculative_tokens=num_speculative_tokens
|
||||
)
|
||||
if speculative_method is not None:
|
||||
spec_kwargs["method"] = speculative_method
|
||||
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
|
||||
spec_kwargs["prompt_lookup_min"] = 1
|
||||
speculative_config = SpeculativeConfig(**spec_kwargs)
|
||||
|
||||
ec_transfer_config = (
|
||||
ECTransferConfig(
|
||||
|
||||
Reference in New Issue
Block a user