[Bugfix] Validate NIXL speculative config compatibility (#49230)
Signed-off-by: Tzu-Ling <tzulingk@nvidia.com> Co-authored-by: GPT-5.6 Sol <noreply@openai.com> Co-authored-by: Cursor Grok 4.5 <noreply@cursor.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
GPT-5.6 Sol
Cursor Grok 4.5
Claude Opus 4.8
parent
65cf1276a2
commit
952694e384
@@ -41,8 +41,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# 4: Add KV block lease renewal through heartbeats
|
||||
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
|
||||
# clock-sync timestamp
|
||||
# 6: Validate EAGLE/MTP speculative configuration compatibility
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 5
|
||||
NIXL_CONNECTOR_VERSION: int = 6
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -78,6 +79,49 @@ class NixlHandshakePayload(KVConnectorHandshakeMetadata):
|
||||
agent_metadata_bytes: bytes # NixlAgentMetadata encoded
|
||||
|
||||
|
||||
def _get_speculative_compatibility_factors(
|
||||
vllm_config: VllmConfig,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return NIXL compatibility factors for hidden-state-based speculators."""
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or not speculative_config.use_eagle():
|
||||
return None
|
||||
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
assert draft_model_config is not None
|
||||
auxiliary_layer_ids = getattr(
|
||||
draft_model_config.hf_config,
|
||||
"eagle_aux_hidden_state_layer_ids",
|
||||
None,
|
||||
)
|
||||
|
||||
# kv_cache_dtype is a user override that defaults to None, meaning "inherit
|
||||
# the target's --kv-cache-dtype". Resolve it to the effective value so an
|
||||
# explicit setting on one side and inheritance on the other (same effective
|
||||
# dtype) don't spuriously mismatch.
|
||||
kv_cache_dtype = (
|
||||
speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype
|
||||
)
|
||||
|
||||
# Note: the draft attention_backend is intentionally not hashed. Its only
|
||||
# transfer-relevant effect is the KV block layout/size, which is validated
|
||||
# per region at runtime in _validate_remote_agent_handshake. The connector
|
||||
# only sees the raw override here (usually None = auto-select), never the
|
||||
# resolved backend, so hashing it would cause false mismatches without
|
||||
# catching anything the runtime layout check misses.
|
||||
return {
|
||||
"method": speculative_config.method,
|
||||
"model": draft_model_config.model,
|
||||
"revision": draft_model_config.revision,
|
||||
"code_revision": draft_model_config.code_revision,
|
||||
"parallel_drafting": speculative_config.parallel_drafting,
|
||||
"kv_cache_dtype": str(kv_cache_dtype),
|
||||
"auxiliary_layer_ids": (
|
||||
tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def compute_nixl_compatibility_hash(
|
||||
vllm_config: VllmConfig, attn_backend_name: str, cross_layers_blocks: bool
|
||||
) -> str:
|
||||
@@ -92,6 +136,7 @@ def compute_nixl_compatibility_hash(
|
||||
- Model architecture (name, dtype, KV heads, layers)
|
||||
- KV cache format (dtype, sliding window)
|
||||
- Attention backend
|
||||
- EAGLE/MTP configuration that affects transferred state
|
||||
|
||||
Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
|
||||
are validated at runtime in _validate_remote_agent_handshake and are not
|
||||
@@ -125,6 +170,7 @@ def compute_nixl_compatibility_hash(
|
||||
"cache_dtype": str(cache_config.cache_dtype),
|
||||
"cross_layers_blocks": cross_layers_blocks,
|
||||
"is_hma_enabled": is_hma_enabled,
|
||||
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
|
||||
}
|
||||
|
||||
compat_hash = hash_factors(factors)
|
||||
|
||||
Reference in New Issue
Block a user