[Core] Make prefix-cache NONE_HASH deterministic by default (#51875)
Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
8f4a7f45c5
commit
ef47a897e2
@@ -15,7 +15,7 @@ from typing import Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.hashing import sha256_cbor, xxhash_cbor
|
||||
from vllm.utils.hashing import xxhash, xxhash_cbor
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
@@ -87,32 +87,73 @@ logger = init_logger(__name__)
|
||||
|
||||
# The hash seed for the first block of any prefix block sequence.
|
||||
#
|
||||
# We use a random value to avoid hash collisions or PYTHONHASHSEED environment
|
||||
# variable if set such that processes can share the seed if needed. This aligns
|
||||
# with the behavior of Python's hash() function, which also uses a random seed
|
||||
# if PYTHONHASHSEED is not set.
|
||||
# For cryptographic hash algorithms it is derived deterministically from a fixed
|
||||
# default seed, so independent vLLM processes compute identical block hashes for
|
||||
# identical content and can share a prefix cache (e.g. KV cache reuse across
|
||||
# nodes) without extra configuration. This does not weaken collision resistance,
|
||||
# which for SHA-256 does not depend on keeping the seed secret; ``cache_salt``
|
||||
# remains the mechanism for intentional cache isolation.
|
||||
#
|
||||
# Non-cryptographic algorithms keep a per-process random seed, because a
|
||||
# predictable seed would let an attacker precompute colliding blocks offline
|
||||
# (see #12621). Setting PYTHONHASHSEED overrides the seed in both cases.
|
||||
#
|
||||
# The function `init_none_hash` initializes this variable globally.
|
||||
NONE_HASH: BlockHash
|
||||
_CBOR_HASH_FUNCTIONS = frozenset({sha256_cbor, xxhash_cbor})
|
||||
|
||||
# Fixed seed used when the PYTHONHASHSEED environment variable is not set and
|
||||
# the hash algorithm is cryptographic.
|
||||
DEFAULT_NONE_HASH_SEED = "vllm-none-hash"
|
||||
|
||||
# Algorithms that are not collision resistant, so the seed must stay secret.
|
||||
_NON_CRYPTO_HASH_FUNCTIONS = frozenset({xxhash, xxhash_cbor})
|
||||
|
||||
# The seed NONE_HASH was derived from, set by init_none_hash.
|
||||
_NONE_HASH_SEED: str | None = None
|
||||
|
||||
|
||||
def resolve_none_hash_seed(hash_fn: Callable[[Any], bytes]) -> str:
|
||||
"""Resolve the seed to derive NONE_HASH from.
|
||||
|
||||
PYTHONHASHSEED wins if set. Otherwise cryptographic algorithms get the
|
||||
fixed default (shareable across processes) and non-cryptographic ones get
|
||||
fresh random bytes, keeping the seed unpredictable where collision
|
||||
resistance depends on it.
|
||||
"""
|
||||
hash_seed = os.getenv("PYTHONHASHSEED")
|
||||
if hash_seed is not None:
|
||||
return hash_seed
|
||||
if hash_fn in _NON_CRYPTO_HASH_FUNCTIONS:
|
||||
return os.urandom(32).hex()
|
||||
return DEFAULT_NONE_HASH_SEED
|
||||
|
||||
|
||||
def get_none_hash_seed() -> str:
|
||||
"""Return the seed NONE_HASH was derived from.
|
||||
|
||||
Components that must agree on NONE_HASH across processes (the P2P tier
|
||||
advertises this during its connect handshake) read the resolved seed here
|
||||
instead of re-deriving it, so they observe the random seed too. Falls back
|
||||
to the deterministic seed before ``init_none_hash`` has run.
|
||||
"""
|
||||
if _NONE_HASH_SEED is None:
|
||||
return DEFAULT_NONE_HASH_SEED
|
||||
return _NONE_HASH_SEED
|
||||
|
||||
|
||||
def init_none_hash(hash_fn: Callable[[Any], bytes]):
|
||||
global NONE_HASH
|
||||
global NONE_HASH, _NONE_HASH_SEED
|
||||
|
||||
hash_seed = os.getenv("PYTHONHASHSEED")
|
||||
if hash_seed is None and hash_fn in _CBOR_HASH_FUNCTIONS:
|
||||
_NONE_HASH_SEED = resolve_none_hash_seed(hash_fn)
|
||||
if hash_fn in _NON_CRYPTO_HASH_FUNCTIONS and os.getenv("PYTHONHASHSEED") is None:
|
||||
logger.warning(
|
||||
"PYTHONHASHSEED is not set. This will lead to non-reproducible "
|
||||
"block-hashes when using CBOR-based hash functions such as "
|
||||
"sha256_cbor or xxhash_cbor. Consider setting PYTHONHASHSEED to a "
|
||||
"fixed value for reproducibility."
|
||||
"Using a random per-process NONE_HASH seed because %s is not "
|
||||
"collision resistant. Block hashes are therefore not reproducible "
|
||||
"across processes; set PYTHONHASHSEED to a shared value to reuse "
|
||||
"the prefix cache across instances, or use sha256.",
|
||||
hash_fn.__name__,
|
||||
)
|
||||
|
||||
if hash_seed is None:
|
||||
NONE_HASH = BlockHash(os.urandom(32))
|
||||
else:
|
||||
NONE_HASH = BlockHash(hash_fn(hash_seed))
|
||||
NONE_HASH = BlockHash(hash_fn(_NONE_HASH_SEED))
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
|
||||
Reference in New Issue
Block a user