[Bugfix][MRV2][Spec Decode] Honor EAGLE3 single-state draft configuration (#61563)
Signed-off-by: Sandeep Agrawal <sandeepa@nvidia.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Kevin H. Luu
parent
06265beb25
commit
21e207efa1
@@ -11,11 +11,36 @@ from vllm.model_executor.models.mimo import MiMoModel
|
||||
from vllm.models.minimax_m3.nvidia import model as minimax_m3
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
requires_aux_hidden_states,
|
||||
reserve_aux_intermediate_tensor_slots,
|
||||
verify_supports_aux_hidden_states_over_pp,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "hf_config", "expected"),
|
||||
[
|
||||
("eagle3", {"eagle_config": {"use_aux_hidden_state": False}}, False),
|
||||
("eagle3", {"eagle_config": {"use_aux_hidden_state": True}}, True),
|
||||
("eagle3", {"eagle_config": {}}, True),
|
||||
("eagle3", {"eagle_config": None}, True),
|
||||
("eagle3", {}, True),
|
||||
("dflash", {"eagle_config": {"use_aux_hidden_state": False}}, True),
|
||||
("dspark", {"eagle_config": {"use_aux_hidden_state": False}}, True),
|
||||
("extract_hidden_states", {}, True),
|
||||
("mtp", {}, False),
|
||||
],
|
||||
)
|
||||
def test_aux_hidden_state_requirement_respects_eagle3_config(
|
||||
method, hf_config, expected
|
||||
):
|
||||
spec_config = SimpleNamespace(
|
||||
method=method,
|
||||
draft_model_config=SimpleNamespace(hf_config=SimpleNamespace(**hf_config)),
|
||||
)
|
||||
assert requires_aux_hidden_states(spec_config) is expected
|
||||
|
||||
|
||||
def test_aux_layers_are_sorted_and_deduplicated():
|
||||
model = EagleModelMixin()
|
||||
model._set_aux_hidden_state_layers((48, 3, 90, 24, 48))
|
||||
|
||||
Reference in New Issue
Block a user