[Bugfix][Core] Fix num_output_placeholders underflow with async scheduling + spec decode (#46066)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ting SUN
2026-07-06 13:50:38 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 641cb59592
commit 373eb314af
3 changed files with 58 additions and 3 deletions
+7 -1
View File
@@ -54,6 +54,7 @@ def create_scheduler(
block_size: int = 16,
max_model_len: int | None = None,
num_speculative_tokens: int | None = None,
speculative_method: str | None = None,
skip_tokenizer_init: bool = False,
async_scheduling: bool = False,
pipeline_parallel_size: int = 1,
@@ -126,9 +127,14 @@ def create_scheduler(
speculative_config: SpeculativeConfig | None = None
if num_speculative_tokens is not None:
speculative_config = SpeculativeConfig(
spec_kwargs: dict = dict(
model="ngram", num_speculative_tokens=num_speculative_tokens
)
if speculative_method is not None:
spec_kwargs["method"] = speculative_method
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
spec_kwargs["prompt_lookup_min"] = 1
speculative_config = SpeculativeConfig(**spec_kwargs)
ec_transfer_config = (
ECTransferConfig(