[Bugfix]Fix FlashInfer warmup crash: autotune(tuning_buckets=...) must round M up, not down (Invalid MXFP8 split-K tactic with spec-decode) (#58165)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yongye Zhu
Claude Opus 5.5
parent
554340f3d3
commit
fba462e008
@@ -363,13 +363,32 @@ def _run_flashinfer_autotune_dummy_runs(
|
||||
|
||||
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
|
||||
for num_tokens in _flashinfer_autotune_token_counts(runner, include_bf16=False):
|
||||
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
|
||||
# The drafter may use more tokens than this pass in the same dummy run.
|
||||
# Include its max M when building the autotune buckets.
|
||||
max_tuning_tokens = num_tokens
|
||||
speculator = getattr(runner, "speculator", None)
|
||||
if speculator is not None:
|
||||
num_query_per_req = speculator.num_query_per_req
|
||||
num_draft_reqs = min(
|
||||
num_tokens,
|
||||
runner.max_num_reqs,
|
||||
runner.max_num_tokens // num_query_per_req,
|
||||
)
|
||||
max_tuning_tokens = max(
|
||||
max_tuning_tokens, num_draft_reqs * num_query_per_req
|
||||
)
|
||||
|
||||
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(
|
||||
max_tuning_tokens
|
||||
)
|
||||
logger.info(
|
||||
"Running FlashInfer autotune with %d tokens and token buckets %s.",
|
||||
num_tokens,
|
||||
tuning_buckets,
|
||||
)
|
||||
with fi_utils.autotune(tuning_buckets=tuning_buckets):
|
||||
# Round M up to match serving-time bucket selection for non-bucket M.
|
||||
# See https://github.com/flashinfer-ai/flashinfer/issues/5450
|
||||
with fi_utils.autotune(tuning_buckets=tuning_buckets, round_up=True):
|
||||
runner._dummy_run(
|
||||
num_tokens=num_tokens,
|
||||
skip_eplb=True,
|
||||
|
||||
Reference in New Issue
Block a user