From c628a93a64fb4929c3c11d8e2c7244c4826b4f76 Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Tue, 19 May 2026 23:15:57 -0400 Subject: [PATCH] [Perf][Bugfix] Update dflash aux layer indexing (#40727) Signed-off-by: Benjamin Chislett --- tests/v1/e2e/spec_decode/test_spec_decode.py | 7 +++++-- vllm/transformers_utils/configs/speculators/algos.py | 3 ++- vllm/v1/worker/gpu_model_runner.py | 5 ++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index f820008670..a9092bb766 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -1315,7 +1315,7 @@ def test_dflash_acceptance_rates(dflash_config): expected_acceptance_lengths = { "mt-bench": 4.24, "humaneval": 6.50, - "gsm8k": 6.54 * 0.95, # runs with a subset of prompts so extra wide tol here + "gsm8k": 6.54 * 0.975, # runs with a subset of prompts so extra wide tol here } tokenizer = spec_llm.get_tokenizer() @@ -1347,7 +1347,10 @@ def test_dflash_acceptance_rates(dflash_config): acceptance_lengths.append(acceptance_len) mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths) - expected_len = expected_len * 0.9 + # Fairly tight tolerance of 95% against the paper's figures, + # watching for regressions. Can be relaxed if test is flaky but be sure to + # check for genuine issues such as #40727. + expected_len = expected_len * 0.95 print( f"DFlash acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}" f" (expected at least {expected_len:.2f})" diff --git a/vllm/transformers_utils/configs/speculators/algos.py b/vllm/transformers_utils/configs/speculators/algos.py index 1bf6960bf6..650f09c39f 100644 --- a/vllm/transformers_utils/configs/speculators/algos.py +++ b/vllm/transformers_utils/configs/speculators/algos.py @@ -99,7 +99,8 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None: aux_layer_ids = config_dict["aux_hidden_state_layer_ids"] pre_trained_config["eagle_aux_hidden_state_layer_ids"] = aux_layer_ids + # DFlash configs use different indexing for the target layers, see #40727 pre_trained_config["dflash_config"] = { "mask_token_id": config_dict["mask_token_id"], - "target_layer_ids": aux_layer_ids, + "target_layer_ids": [i - 1 for i in aux_layer_ids], } diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 61e705ceb7..5d2c40e006 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -5213,7 +5213,10 @@ class GPUModelRunner( eagle_config = getattr(hf_config, "eagle_config", None) if dflash_config and isinstance(dflash_config, dict): - layer_ids = dflash_config.get("target_layer_ids") + # Add 1 to convert DFlash's aux layer id semantics + layer_ids = [ + i + 1 for i in (dflash_config.get("target_layer_ids") or []) + ] if eagle_config and isinstance(eagle_config, dict): layer_ids = eagle_config.get("eagle_aux_hidden_state_layer_ids")