[Bugfix][Security] Fix concurrent sparse invariant race bypassing CVE remediation (#48583)

Signed-off-by: jperezde <jperezde@redhat.com>
This commit is contained in:
Juan Pérez de Algaba
2026-07-14 11:08:24 +00:00
committed by GitHub
parent 50ac1c7bab
commit 793cf79c89
5 changed files with 298 additions and 13 deletions
@@ -0,0 +1,263 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import io
import threading
from concurrent.futures import ThreadPoolExecutor
import pybase64
import pytest
import torch
from vllm.config import ModelConfig
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
from vllm.renderers.embed_utils import safe_load_prompt_embeds
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
def _encode_tensor(tensor: torch.Tensor) -> bytes:
buffer = io.BytesIO()
torch.save(tensor, buffer)
buffer.seek(0)
return pybase64.b64encode(buffer.read())
def _create_malicious_sparse_tensor() -> torch.Tensor:
indices = torch.tensor([[10], [10]])
values = torch.tensor([1.0])
shape = (3, 3)
return torch.sparse_coo_tensor(
indices, values, shape, dtype=torch.float32, check_invariants=False
)
def _create_valid_dense_tensor(hidden_size: int = 768) -> torch.Tensor:
return torch.randn(4, hidden_size, dtype=torch.float32)
@pytest.fixture
def model_config():
return ModelConfig(
model="facebook/opt-125m",
tokenizer="facebook/opt-125m",
tokenizer_mode="auto",
trust_remote_code=False,
dtype="float32",
seed=0,
enable_prompt_embeds=True,
)
class TestNegativeControlWithoutRace:
"""The invalid payload must be rejected even without concurrency."""
def test_malicious_sparse_rejected_by_prompt_loader(self, model_config):
encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
safe_load_prompt_embeds(model_config, encoded)
def test_malicious_sparse_rejected_by_image_loader(self):
io_handler = ImageEmbeddingMediaIO()
encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
io_handler.load_base64("", encoded.decode("utf-8"))
def test_malicious_sparse_rejected_by_audio_loader(self):
io_handler = AudioEmbeddingMediaIO()
encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
io_handler.load_base64("", encoded.decode("utf-8"))
class TestConcurrentRaceProtection:
"""Verify the lock prevents the A-enter, B-enter, A-exit, B-load race."""
def test_forced_interleaving_still_rejects_invalid(self, model_config):
"""Force the exact interleaving that bypasses the guard without a lock.
Thread A (benign): enters context, loads valid tensor, exits context.
Thread B (malicious): enters context after A, but loads after A exits.
Without _SPARSE_LOAD_LOCK the flag would be False when B loads.
With the lock, B cannot enter until A fully completes.
"""
barrier = threading.Barrier(2, timeout=5)
results: dict[str, object] = {}
valid_encoded = _encode_tensor(_create_valid_dense_tensor())
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
def thread_a_benign():
"""Enter context, signal B, load, exit."""
try:
with check_sparse_tensor_invariants_threadsafe():
barrier.wait() # signal B that A holds the lock
tensor = torch.load(
io.BytesIO(pybase64.b64decode(valid_encoded, validate=True)),
weights_only=True,
map_location=torch.device("cpu"),
)
results["a_loaded"] = True
results["a_tensor_valid"] = isinstance(tensor, torch.Tensor)
except threading.BrokenBarrierError:
results["a_barrier_broken"] = True
def thread_b_malicious():
"""Wait for A to hold the lock, then try to acquire it."""
try:
barrier.wait() # wait until A is inside the lock
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(
io.BytesIO(
pybase64.b64decode(malicious_encoded, validate=True)
),
weights_only=True,
map_location=torch.device("cpu"),
)
tensor.to_dense()
results["b_bypass"] = True
except (RuntimeError, ValueError):
results["b_rejected"] = True
except threading.BrokenBarrierError:
results["b_barrier_broken"] = True
t_a = threading.Thread(target=thread_a_benign)
t_b = threading.Thread(target=thread_b_malicious)
t_a.start()
t_b.start()
t_a.join(timeout=10)
t_b.join(timeout=10)
assert results.get("a_loaded") is True
assert results.get("a_tensor_valid") is True
assert results.get("b_rejected") is True, (
"Malicious tensor was NOT rejected under concurrency"
)
assert results.get("b_bypass") is not True
def test_concurrent_loads_all_reject_invalid(self, model_config):
"""Multiple concurrent malicious loads must all be rejected."""
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
num_workers = 4
rejected = []
bypassed = []
def attempt_load(_):
try:
safe_load_prompt_embeds(model_config, malicious_encoded)
bypassed.append(True)
except (RuntimeError, ValueError):
rejected.append(True)
with ThreadPoolExecutor(max_workers=num_workers) as pool:
list(pool.map(attempt_load, range(num_workers)))
assert len(rejected) == num_workers
assert len(bypassed) == 0
def test_concurrent_valid_loads_succeed(self, model_config):
"""Concurrent valid loads must all succeed (no false rejections)."""
valid_encoded = _encode_tensor(_create_valid_dense_tensor())
num_workers = 4
successes = []
failures = []
def attempt_load(_):
try:
result = safe_load_prompt_embeds(model_config, valid_encoded)
successes.append(result.shape)
except Exception as e:
failures.append(str(e))
with ThreadPoolExecutor(max_workers=num_workers) as pool:
list(pool.map(attempt_load, range(num_workers)))
assert len(successes) == num_workers
assert len(failures) == 0
class TestGlobalFlagRestoration:
"""The global invariant flag must be correctly restored after use."""
def test_flag_restored_after_success(self, model_config):
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
valid_encoded = _encode_tensor(_create_valid_dense_tensor())
safe_load_prompt_embeds(model_config, valid_encoded)
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
def test_flag_restored_after_exception(self, model_config):
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
safe_load_prompt_embeds(model_config, malicious_encoded)
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
def test_flag_restored_after_concurrent_exceptions(self, model_config):
"""Flag must be restored even when multiple threads raise."""
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
def attempt_load(_):
with contextlib.suppress(RuntimeError, ValueError):
safe_load_prompt_embeds(model_config, malicious_encoded)
with ThreadPoolExecutor(max_workers=4) as pool:
list(pool.map(attempt_load, range(4)))
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
class TestCrossLoaderLockSharing:
"""All loaders must share the same lock to prevent cross-loader races."""
def test_prompt_and_image_share_lock(self, model_config):
"""Prompt and image loaders cannot run their sparse guards
concurrently."""
valid_prompt = _encode_tensor(_create_valid_dense_tensor())
valid_image = _encode_tensor(_create_valid_dense_tensor(hidden_size=10))
io_handler = ImageEmbeddingMediaIO()
def load_prompt():
safe_load_prompt_embeds(model_config, valid_prompt)
def load_image():
io_handler.load_base64("", valid_image.decode("utf-8"))
# If both threads could hold the lock simultaneously, the barrier
# would succeed. Since the lock serializes them, the barrier will
# time out for the second thread (proving mutual exclusion).
t1 = threading.Thread(target=load_prompt)
t2 = threading.Thread(target=load_image)
t1.start()
t2.start()
t1.join(timeout=10)
t2.join(timeout=10)
# The key assertion: both completed (barrier didn't deadlock due to
# serialization — one finishes before the other starts), AND invalid
# tensors are still rejected.
malicious = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
safe_load_prompt_embeds(model_config, malicious)
with pytest.raises((RuntimeError, ValueError)):
io_handler.load_base64("", malicious.decode("utf-8"))
def test_all_loaders_use_same_context_manager(self):
"""Verify all modules reference the same context manager function."""
import vllm.multimodal.media.audio as audio_mod
import vllm.multimodal.media.image as image_mod
import vllm.renderers.embed_utils as embed_mod
from vllm.utils.sparse_utils import (
check_sparse_tensor_invariants_threadsafe as cm_from_utils,
)
assert embed_mod.check_sparse_tensor_invariants_threadsafe is cm_from_utils
assert image_mod.check_sparse_tensor_invariants_threadsafe is cm_from_utils
assert audio_mod.check_sparse_tensor_invariants_threadsafe is cm_from_utils
+3 -6
View File
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
from vllm.multimodal.audio import resample_audio_pyav
from vllm.utils.import_utils import PlaceholderModule
from vllm.utils.serial_utils import tensor2base64
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
from .base import MediaIO
@@ -282,9 +283,7 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
def load_bytes(self, data: bytes) -> torch.Tensor:
buffer = BytesIO(data)
# Enable sparse tensor integrity checks to prevent out-of-bounds
# writes from maliciously crafted tensors
with torch.sparse.check_sparse_tensor_invariants():
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(buffer, weights_only=True)
return tensor.to_dense()
@@ -292,9 +291,7 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
return self.load_bytes(pybase64.b64decode(data, validate=True))
def load_file(self, filepath: Path) -> torch.Tensor:
# Enable sparse tensor integrity checks to prevent out-of-bounds
# writes from maliciously crafted tensors
with torch.sparse.check_sparse_tensor_invariants():
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(filepath, weights_only=True)
return tensor.to_dense()
+3 -4
View File
@@ -11,6 +11,7 @@ from PIL import Image
import vllm.envs as envs
from vllm.utils.serial_utils import tensor2base64
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
from ..image import convert_image_mode, normalize_image, rgba_to_rgb
from .base import MediaIO, MediaWithBytes
@@ -123,9 +124,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
def _load_pickled_torch(self, data: bytes) -> torch.Tensor:
buffer = BytesIO(data)
# Enable sparse tensor integrity checks to prevent out-of-bounds
# writes from maliciously crafted tensors
with torch.sparse.check_sparse_tensor_invariants():
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(buffer, weights_only=True)
return tensor.to_dense()
@@ -146,7 +145,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
if filepath.suffix == ".npy":
return torch.from_numpy(np.load(filepath))
with torch.sparse.check_sparse_tensor_invariants():
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(filepath, weights_only=True)
return tensor.to_dense()
+2 -3
View File
@@ -8,6 +8,7 @@ import torch
from vllm.exceptions import VLLMValidationError
from vllm.utils.async_utils import make_async
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
if TYPE_CHECKING:
from vllm.config import ModelConfig
@@ -23,9 +24,7 @@ def safe_load_prompt_embeds(
parameter="prompt_embeds",
)
# Enable sparse tensor integrity checks to prevent out-of-bounds
# writes from maliciously crafted tensors
with torch.sparse.check_sparse_tensor_invariants():
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(
BytesIO(pybase64.b64decode(embed, validate=True)),
weights_only=True,
+27
View File
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Thread-safe wrapper for sparse tensor invariant validation.
PyTorch's `torch.sparse.check_sparse_tensor_invariants()` context manager
manipulates a **process-global** flag (save/enable/restore). When multiple
embedding-load operations run concurrently on a thread-pool executor, one
context can restore the flag to `False` while another thread is still inside
its guard, bypassing the invariant check.
All call sites MUST use `check_sparse_tensor_invariants_threadsafe()`
which serializes access behind a lock.
"""
import contextlib
import threading
import torch
_SPARSE_LOAD_LOCK = threading.Lock()
@contextlib.contextmanager
def check_sparse_tensor_invariants_threadsafe():
with _SPARSE_LOAD_LOCK, torch.sparse.check_sparse_tensor_invariants():
yield