[Bugfix] Validate NIXL speculative config compatibility (#49230)
Signed-off-by: Tzu-Ling <tzulingk@nvidia.com> Co-authored-by: GPT-5.6 Sol <noreply@openai.com> Co-authored-by: Cursor Grok 4.5 <noreply@cursor.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
GPT-5.6 Sol
Cursor Grok 4.5
Claude Opus 4.8
parent
65cf1276a2
commit
952694e384
@@ -57,7 +57,9 @@ th:not(:first-child) {
|
|||||||
| Multimodal | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ |
|
| Multimodal | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ |
|
||||||
| Encoder-Decoder | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
| Encoder-Decoder | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||||
|
|
||||||
<sup>1</sup> P and D instances must use the same speculation configuration.
|
<sup>1</sup> P and D instances must use compatible speculation configurations.
|
||||||
|
See [Configuration Notes](#configuration-notes) below for what must match and
|
||||||
|
what may differ.
|
||||||
|
|
||||||
<sup>2</sup> Requires `FLASH_ATTN` or `FLASHINFER` backend **and** `HND` KV cache layout. Enable via `--kv-transfer-config '{"kv_connector_extra_config": {"enable_cross_layers_blocks": "True"}}'`.
|
<sup>2</sup> Requires `FLASH_ATTN` or `FLASHINFER` backend **and** `HND` KV cache layout. Enable via `--kv-transfer-config '{"kv_connector_extra_config": {"enable_cross_layers_blocks": "True"}}'`.
|
||||||
|
|
||||||
@@ -79,6 +81,7 @@ By default, a **compatibility hash** is checked during handshake. P and D instan
|
|||||||
- Model (architecture, dtype, number of KV heads, head size, number of hidden layers)
|
- Model (architecture, dtype, number of KV heads, head size, number of hidden layers)
|
||||||
- Attention backend
|
- Attention backend
|
||||||
- KV cache dtype (`cache_dtype`)
|
- KV cache dtype (`cache_dtype`)
|
||||||
|
- EAGLE/MTP-style speculative method and draft-model configuration
|
||||||
|
|
||||||
!!! warning
|
!!! warning
|
||||||
Disable the hash check with `--kv-transfer-config '{"kv_connector_extra_config": {"enforce_handshake_compat": false}}'` at your own risk.
|
Disable the hash check with `--kv-transfer-config '{"kv_connector_extra_config": {"enforce_handshake_compat": false}}'` at your own risk.
|
||||||
@@ -88,6 +91,10 @@ By default, a **compatibility hash** is checked during handshake. P and D instan
|
|||||||
- `tensor-parallel-size` (heterogeneous TP, subject to model restrictions above)
|
- `tensor-parallel-size` (heterogeneous TP, subject to model restrictions above)
|
||||||
- `block-size` (heterogeneous block size, subject to restrictions above)
|
- `block-size` (heterogeneous block size, subject to restrictions above)
|
||||||
- Number of KV cache blocks (determined by available memory on each instance)
|
- Number of KV cache blocks (determined by available memory on each instance)
|
||||||
|
- `num_speculative_tokens` (prefill and decode may use different draft depths)
|
||||||
|
- Draft-model `attention_backend` (each instance auto-selects independently; the
|
||||||
|
resulting KV block layout is validated at handshake time rather than via the
|
||||||
|
compatibility hash)
|
||||||
|
|
||||||
### KV cache layout
|
### KV cache layout
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import textwrap
|
|||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from types import SimpleNamespace
|
||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
@@ -2796,6 +2797,115 @@ def test_failed_request_skips_kv_postprocessing(
|
|||||||
assert invalid_blocks == {1, 2, 3}
|
assert invalid_blocks == {1, 2, 3}
|
||||||
|
|
||||||
|
|
||||||
|
def _set_test_speculative_config(
|
||||||
|
vllm_config,
|
||||||
|
*,
|
||||||
|
method: str = "eagle3",
|
||||||
|
model: str = "test/eagle3-drafter",
|
||||||
|
revision: str | None = None,
|
||||||
|
code_revision: str | None = None,
|
||||||
|
num_speculative_tokens: int = 1,
|
||||||
|
parallel_drafting: bool = False,
|
||||||
|
kv_cache_dtype: str | None = None,
|
||||||
|
attention_backend: str | None = None,
|
||||||
|
auxiliary_layer_ids: tuple[int, ...] = (2, 16, 29),
|
||||||
|
) -> None:
|
||||||
|
draft_model_config = SimpleNamespace(
|
||||||
|
model=model,
|
||||||
|
revision=revision,
|
||||||
|
code_revision=code_revision,
|
||||||
|
hf_config=SimpleNamespace(eagle_aux_hidden_state_layer_ids=auxiliary_layer_ids),
|
||||||
|
)
|
||||||
|
vllm_config.speculative_config = SimpleNamespace(
|
||||||
|
method=method,
|
||||||
|
draft_model_config=draft_model_config,
|
||||||
|
num_speculative_tokens=num_speculative_tokens,
|
||||||
|
parallel_drafting=parallel_drafting,
|
||||||
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
|
attention_backend=attention_backend,
|
||||||
|
use_eagle=lambda: True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"remote_overrides,should_match",
|
||||||
|
[
|
||||||
|
({}, True),
|
||||||
|
({"num_speculative_tokens": 2}, True),
|
||||||
|
({"method": "mtp"}, False),
|
||||||
|
({"model": "test/different-drafter"}, False),
|
||||||
|
({"revision": "different-revision"}, False),
|
||||||
|
({"parallel_drafting": True}, False),
|
||||||
|
({"kv_cache_dtype": "fp8"}, False),
|
||||||
|
# attention_backend is intentionally not part of the compat hash
|
||||||
|
# (see _get_speculative_compatibility_factors); overriding it must
|
||||||
|
# not change the hash.
|
||||||
|
({"attention_backend": "FLASHINFER"}, True),
|
||||||
|
({"auxiliary_layer_ids": (2, 16, 30)}, False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
def test_speculative_config_compatibility_hash(
|
||||||
|
remote_overrides: dict[str, Any], should_match: bool
|
||||||
|
):
|
||||||
|
local_config = create_vllm_config()
|
||||||
|
remote_config = create_vllm_config()
|
||||||
|
_set_test_speculative_config(local_config)
|
||||||
|
_set_test_speculative_config(remote_config, **remote_overrides)
|
||||||
|
|
||||||
|
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
|
||||||
|
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
|
||||||
|
|
||||||
|
assert (local_hash == remote_hash) is should_match
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
def test_missing_speculative_config_changes_compatibility_hash():
|
||||||
|
regular_config = create_vllm_config()
|
||||||
|
speculative_config = create_vllm_config()
|
||||||
|
_set_test_speculative_config(speculative_config)
|
||||||
|
|
||||||
|
regular_hash = compute_nixl_compatibility_hash(regular_config, "FLASH_ATTN", False)
|
||||||
|
speculative_hash = compute_nixl_compatibility_hash(
|
||||||
|
speculative_config, "FLASH_ATTN", False
|
||||||
|
)
|
||||||
|
|
||||||
|
assert regular_hash != speculative_hash
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
def test_speculative_kv_cache_dtype_resolves_to_target():
|
||||||
|
# The draft kv_cache_dtype override defaults to None ("inherit the target's
|
||||||
|
# --kv-cache-dtype"). An explicit setting on one side that matches the
|
||||||
|
# other side's inherited (resolved) dtype must not spuriously mismatch.
|
||||||
|
local_config = create_vllm_config(cache_dtype="fp8")
|
||||||
|
remote_config = create_vllm_config(cache_dtype="fp8")
|
||||||
|
_set_test_speculative_config(local_config, kv_cache_dtype="fp8") # explicit
|
||||||
|
_set_test_speculative_config(remote_config, kv_cache_dtype=None) # inherits
|
||||||
|
|
||||||
|
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
|
||||||
|
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
|
||||||
|
|
||||||
|
assert local_hash == remote_hash
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
def test_speculative_attention_backend_not_in_compatibility_hash():
|
||||||
|
# The draft attention_backend is intentionally excluded from the hash: the
|
||||||
|
# connector only has the raw override (auto-select), and its transfer-
|
||||||
|
# relevant effect is validated per region at runtime. Differing overrides
|
||||||
|
# must not change the hash.
|
||||||
|
local_config = create_vllm_config()
|
||||||
|
remote_config = create_vllm_config()
|
||||||
|
_set_test_speculative_config(local_config, attention_backend=None)
|
||||||
|
_set_test_speculative_config(remote_config, attention_backend="FLASHINFER")
|
||||||
|
|
||||||
|
local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN", False)
|
||||||
|
remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN", False)
|
||||||
|
|
||||||
|
assert local_hash == remote_hash
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"mismatch_type,config_overrides,version_override,should_fail,enforce_handshake_compat",
|
"mismatch_type,config_overrides,version_override,should_fail,enforce_handshake_compat",
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -41,8 +41,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
|||||||
# 4: Add KV block lease renewal through heartbeats
|
# 4: Add KV block lease renewal through heartbeats
|
||||||
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
|
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
|
||||||
# clock-sync timestamp
|
# clock-sync timestamp
|
||||||
|
# 6: Validate EAGLE/MTP speculative configuration compatibility
|
||||||
#
|
#
|
||||||
NIXL_CONNECTOR_VERSION: int = 5
|
NIXL_CONNECTOR_VERSION: int = 6
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -78,6 +79,49 @@ class NixlHandshakePayload(KVConnectorHandshakeMetadata):
|
|||||||
agent_metadata_bytes: bytes # NixlAgentMetadata encoded
|
agent_metadata_bytes: bytes # NixlAgentMetadata encoded
|
||||||
|
|
||||||
|
|
||||||
|
def _get_speculative_compatibility_factors(
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
"""Return NIXL compatibility factors for hidden-state-based speculators."""
|
||||||
|
speculative_config = vllm_config.speculative_config
|
||||||
|
if speculative_config is None or not speculative_config.use_eagle():
|
||||||
|
return None
|
||||||
|
|
||||||
|
draft_model_config = speculative_config.draft_model_config
|
||||||
|
assert draft_model_config is not None
|
||||||
|
auxiliary_layer_ids = getattr(
|
||||||
|
draft_model_config.hf_config,
|
||||||
|
"eagle_aux_hidden_state_layer_ids",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
# kv_cache_dtype is a user override that defaults to None, meaning "inherit
|
||||||
|
# the target's --kv-cache-dtype". Resolve it to the effective value so an
|
||||||
|
# explicit setting on one side and inheritance on the other (same effective
|
||||||
|
# dtype) don't spuriously mismatch.
|
||||||
|
kv_cache_dtype = (
|
||||||
|
speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype
|
||||||
|
)
|
||||||
|
|
||||||
|
# Note: the draft attention_backend is intentionally not hashed. Its only
|
||||||
|
# transfer-relevant effect is the KV block layout/size, which is validated
|
||||||
|
# per region at runtime in _validate_remote_agent_handshake. The connector
|
||||||
|
# only sees the raw override here (usually None = auto-select), never the
|
||||||
|
# resolved backend, so hashing it would cause false mismatches without
|
||||||
|
# catching anything the runtime layout check misses.
|
||||||
|
return {
|
||||||
|
"method": speculative_config.method,
|
||||||
|
"model": draft_model_config.model,
|
||||||
|
"revision": draft_model_config.revision,
|
||||||
|
"code_revision": draft_model_config.code_revision,
|
||||||
|
"parallel_drafting": speculative_config.parallel_drafting,
|
||||||
|
"kv_cache_dtype": str(kv_cache_dtype),
|
||||||
|
"auxiliary_layer_ids": (
|
||||||
|
tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def compute_nixl_compatibility_hash(
|
def compute_nixl_compatibility_hash(
|
||||||
vllm_config: VllmConfig, attn_backend_name: str, cross_layers_blocks: bool
|
vllm_config: VllmConfig, attn_backend_name: str, cross_layers_blocks: bool
|
||||||
) -> str:
|
) -> str:
|
||||||
@@ -92,6 +136,7 @@ def compute_nixl_compatibility_hash(
|
|||||||
- Model architecture (name, dtype, KV heads, layers)
|
- Model architecture (name, dtype, KV heads, layers)
|
||||||
- KV cache format (dtype, sliding window)
|
- KV cache format (dtype, sliding window)
|
||||||
- Attention backend
|
- Attention backend
|
||||||
|
- EAGLE/MTP configuration that affects transferred state
|
||||||
|
|
||||||
Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
|
Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
|
||||||
are validated at runtime in _validate_remote_agent_handshake and are not
|
are validated at runtime in _validate_remote_agent_handshake and are not
|
||||||
@@ -125,6 +170,7 @@ def compute_nixl_compatibility_hash(
|
|||||||
"cache_dtype": str(cache_config.cache_dtype),
|
"cache_dtype": str(cache_config.cache_dtype),
|
||||||
"cross_layers_blocks": cross_layers_blocks,
|
"cross_layers_blocks": cross_layers_blocks,
|
||||||
"is_hma_enabled": is_hma_enabled,
|
"is_hma_enabled": is_hma_enabled,
|
||||||
|
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
|
||||||
}
|
}
|
||||||
|
|
||||||
compat_hash = hash_factors(factors)
|
compat_hash = hash_factors(factors)
|
||||||
|
|||||||
Reference in New Issue
Block a user