[Bugfix] Validate NIXL speculative config compatibility (#49230)
Signed-off-by: Tzu-Ling <tzulingk@nvidia.com> Co-authored-by: GPT-5.6 Sol <noreply@openai.com> Co-authored-by: Cursor Grok 4.5 <noreply@cursor.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
GPT-5.6 Sol
Cursor Grok 4.5
Claude Opus 4.8
parent
65cf1276a2
commit
952694e384
@@ -9,6 +9,7 @@ import textwrap
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -2796,6 +2797,115 @@ def test_failed_request_skips_kv_postprocessing(
|
||||
assert invalid_blocks == {1, 2, 3}
|
||||
|
||||
|
||||
def _set_test_speculative_config(
|
||||
vllm_config,
|
||||
*,
|
||||
method: str = "eagle3",
|
||||
model: str = "test/eagle3-drafter",
|
||||
revision: str | None = None,
|
||||
code_revision: str | None = None,
|
||||
num_speculative_tokens: int = 1,
|
||||
parallel_drafting: bool = False,
|
||||
kv_cache_dtype: str | None = None,
|
||||
attention_backend: str | None = None,
|
||||
auxiliary_layer_ids: tuple[int, ...] = (2, 16, 29),
|
||||
) -> None:
|
||||
draft_model_config = SimpleNamespace(
|
||||
model=model,
|
||||
revision=revision,
|
||||
code_revision=code_revision,
|
||||
hf_config=SimpleNamespace(eagle_aux_hidden_state_layer_ids=auxiliary_layer_ids),
|
||||
)
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
method=method,
|
||||
draft_model_config=draft_model_config,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=parallel_drafting,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
attention_backend=attention_backend,
|
||||
use_eagle=lambda: True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"remote_overrides,should_match",
|
||||
[
|
||||
({}, True),
|
||||
({"num_speculative_tokens": 2}, True),
|
||||
({"method": "mtp"}, False),
|
||||
({"model": "test/different-drafter"}, False),
|
||||
({"revision": "different-revision"}, False),
|
||||
({"parallel_drafting": True}, False),
|
||||
({"kv_cache_dtype": "fp8"}, False),
|
||||
# attention_backend is intentionally not part of the compat hash
|
||||
# (see _get_speculative_compatibility_factors); overriding it must
|
||||
# not change the hash.
|
||||
({"attention_backend": "FLASHINFER"}, True),
|
||||
({"auxiliary_layer_ids": (2, 16, 30)}, False),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_speculative_config_compatibility_hash(
|
||||
remote_overrides: dict[str, Any], should_match: bool
|
||||
):
|
||||
local_config = create_vllm_config()
|
||||
remote_config = create_vllm_config()
|
||||
_set_test_speculative_config(local_config)
|
||||
_set_test_speculative_config(remote_config, **remote_overrides)
|
||||
|
||||
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
|
||||
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
|
||||
|
||||
assert (local_hash == remote_hash) is should_match
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_missing_speculative_config_changes_compatibility_hash():
|
||||
regular_config = create_vllm_config()
|
||||
speculative_config = create_vllm_config()
|
||||
_set_test_speculative_config(speculative_config)
|
||||
|
||||
regular_hash = compute_nixl_compatibility_hash(regular_config, "FLASH_ATTN", False)
|
||||
speculative_hash = compute_nixl_compatibility_hash(
|
||||
speculative_config, "FLASH_ATTN", False
|
||||
)
|
||||
|
||||
assert regular_hash != speculative_hash
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_speculative_kv_cache_dtype_resolves_to_target():
|
||||
# The draft kv_cache_dtype override defaults to None ("inherit the target's
|
||||
# --kv-cache-dtype"). An explicit setting on one side that matches the
|
||||
# other side's inherited (resolved) dtype must not spuriously mismatch.
|
||||
local_config = create_vllm_config(cache_dtype="fp8")
|
||||
remote_config = create_vllm_config(cache_dtype="fp8")
|
||||
_set_test_speculative_config(local_config, kv_cache_dtype="fp8") # explicit
|
||||
_set_test_speculative_config(remote_config, kv_cache_dtype=None) # inherits
|
||||
|
||||
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
|
||||
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
|
||||
|
||||
assert local_hash == remote_hash
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_speculative_attention_backend_not_in_compatibility_hash():
|
||||
# The draft attention_backend is intentionally excluded from the hash: the
|
||||
# connector only has the raw override (auto-select), and its transfer-
|
||||
# relevant effect is validated per region at runtime. Differing overrides
|
||||
# must not change the hash.
|
||||
local_config = create_vllm_config()
|
||||
remote_config = create_vllm_config()
|
||||
_set_test_speculative_config(local_config, attention_backend=None)
|
||||
_set_test_speculative_config(remote_config, attention_backend="FLASHINFER")
|
||||
|
||||
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
|
||||
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
|
||||
|
||||
assert local_hash == remote_hash
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mismatch_type,config_overrides,version_override,should_fail,enforce_handshake_compat",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user