[Bugfix] Validate NIXL speculative config compatibility (#49230)

Signed-off-by: Tzu-Ling <tzulingk@nvidia.com>
Co-authored-by: GPT-5.6 Sol <noreply@openai.com>
Co-authored-by: Cursor Grok 4.5 <noreply@cursor.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tzu-Ling Kan
2026-08-04 02:51:04 +08:00
committed by GitHub
co-authored by GPT-5.6 Sol Cursor Grok 4.5 Claude Opus 4.8
parent 65cf1276a2
commit 952694e384
3 changed files with 165 additions and 2 deletions
@@ -9,6 +9,7 @@ import textwrap
import time
import uuid
from collections import defaultdict
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import MagicMock, patch
@@ -2796,6 +2797,115 @@ def test_failed_request_skips_kv_postprocessing(
assert invalid_blocks == {1, 2, 3}
def _set_test_speculative_config(
vllm_config,
*,
method: str = "eagle3",
model: str = "test/eagle3-drafter",
revision: str | None = None,
code_revision: str | None = None,
num_speculative_tokens: int = 1,
parallel_drafting: bool = False,
kv_cache_dtype: str | None = None,
attention_backend: str | None = None,
auxiliary_layer_ids: tuple[int, ...] = (2, 16, 29),
) -> None:
draft_model_config = SimpleNamespace(
model=model,
revision=revision,
code_revision=code_revision,
hf_config=SimpleNamespace(eagle_aux_hidden_state_layer_ids=auxiliary_layer_ids),
)
vllm_config.speculative_config = SimpleNamespace(
method=method,
draft_model_config=draft_model_config,
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=parallel_drafting,
kv_cache_dtype=kv_cache_dtype,
attention_backend=attention_backend,
use_eagle=lambda: True,
)
@pytest.mark.parametrize(
"remote_overrides,should_match",
[
({}, True),
({"num_speculative_tokens": 2}, True),
({"method": "mtp"}, False),
({"model": "test/different-drafter"}, False),
({"revision": "different-revision"}, False),
({"parallel_drafting": True}, False),
({"kv_cache_dtype": "fp8"}, False),
# attention_backend is intentionally not part of the compat hash
# (see _get_speculative_compatibility_factors); overriding it must
# not change the hash.
({"attention_backend": "FLASHINFER"}, True),
({"auxiliary_layer_ids": (2, 16, 30)}, False),
],
)
@pytest.mark.skip_global_cleanup
def test_speculative_config_compatibility_hash(
remote_overrides: dict[str, Any], should_match: bool
):
local_config = create_vllm_config()
remote_config = create_vllm_config()
_set_test_speculative_config(local_config)
_set_test_speculative_config(remote_config, **remote_overrides)
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
assert (local_hash == remote_hash) is should_match
@pytest.mark.skip_global_cleanup
def test_missing_speculative_config_changes_compatibility_hash():
regular_config = create_vllm_config()
speculative_config = create_vllm_config()
_set_test_speculative_config(speculative_config)
regular_hash = compute_nixl_compatibility_hash(regular_config, "FLASH_ATTN", False)
speculative_hash = compute_nixl_compatibility_hash(
speculative_config, "FLASH_ATTN", False
)
assert regular_hash != speculative_hash
@pytest.mark.skip_global_cleanup
def test_speculative_kv_cache_dtype_resolves_to_target():
# The draft kv_cache_dtype override defaults to None ("inherit the target's
# --kv-cache-dtype"). An explicit setting on one side that matches the
# other side's inherited (resolved) dtype must not spuriously mismatch.
local_config = create_vllm_config(cache_dtype="fp8")
remote_config = create_vllm_config(cache_dtype="fp8")
_set_test_speculative_config(local_config, kv_cache_dtype="fp8") # explicit
_set_test_speculative_config(remote_config, kv_cache_dtype=None) # inherits
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
assert local_hash == remote_hash
@pytest.mark.skip_global_cleanup
def test_speculative_attention_backend_not_in_compatibility_hash():
# The draft attention_backend is intentionally excluded from the hash: the
# connector only has the raw override (auto-select), and its transfer-
# relevant effect is validated per region at runtime. Differing overrides
# must not change the hash.
local_config = create_vllm_config()
remote_config = create_vllm_config()
_set_test_speculative_config(local_config, attention_backend=None)
_set_test_speculative_config(remote_config, attention_backend="FLASHINFER")
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
assert local_hash == remote_hash
@pytest.mark.parametrize(
"mismatch_type,config_overrides,version_override,should_fail,enforce_handshake_compat",
[