diff --git a/benchmarks/attention_benchmarks/mla_runner.py b/benchmarks/attention_benchmarks/mla_runner.py index 790b2f8944..8a6ae3f87d 100644 --- a/benchmarks/attention_benchmarks/mla_runner.py +++ b/benchmarks/attention_benchmarks/mla_runner.py @@ -409,10 +409,6 @@ def _build_attention_metadata( seq_lens_cpu = torch.tensor(kv_lens, dtype=torch.int32) seq_lens_gpu = seq_lens_cpu.to(device) - # Build num_computed_tokens (context length for each request) - context_lens = [kv_len - q_len for q_len, kv_len in zip(q_lens, kv_lens)] - num_computed_tokens_cpu = torch.tensor(context_lens, dtype=torch.int32) - # Build block table num_blocks_per_req = [(kv + block_size - 1) // block_size for kv in kv_lens] max_num_blocks = max(num_blocks_per_req) @@ -454,8 +450,6 @@ def _build_attention_metadata( query_start_loc_cpu=q_start_cpu, seq_lens=seq_lens_gpu, seq_lens_cpu_upper_bound=seq_lens_cpu, - _seq_lens_cpu=seq_lens_cpu, - _num_computed_tokens_cpu=num_computed_tokens_cpu, slot_mapping=slot_mapping, block_table_tensor=block_table_gpu, dcp_local_seq_lens=None, diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index f894861022..71b9dbbab8 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -561,24 +561,6 @@ def test_prefix_cache_default(): assert engine_args.prefix_cache_retention_interval == 64 -def test_prefix_cache_retention_interval_from_deprecated_env( - monkeypatch, caplog, disable_log_dedup -): - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64") - - engine_args = EngineArgs() - - assert engine_args.prefix_cache_retention_interval == 64 - assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text - assert "deprecated" in caplog.text - assert "prefix_cache_retention_interval" in caplog.text - - parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) - args = parser.parse_args(["--prefix-cache-retention-interval", "32"]) - engine_args = EngineArgs.from_cli_args(args) - assert engine_args.prefix_cache_retention_interval == 32 - - @pytest.mark.parametrize( ("arg", "expected", "option"), [ diff --git a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py index 99da0af7d5..aabce97fac 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py @@ -81,7 +81,7 @@ def _make_common_metadata(): query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc, seq_lens=seq_lens, - _seq_lens_cpu=seq_lens, + seq_lens_cpu_upper_bound=seq_lens, num_reqs=2, num_actual_tokens=2, max_query_len=1, @@ -99,7 +99,7 @@ def _make_mixed_common_metadata(): query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc, seq_lens=seq_lens, - _seq_lens_cpu=seq_lens, + seq_lens_cpu_upper_bound=seq_lens, num_reqs=2, num_actual_tokens=5, max_query_len=4, diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index 1f6601dc6c..082a60b7a5 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -705,9 +705,13 @@ def test_gfx950_compressed_cache_canonicalizes_nonfinite(writer: str) -> None: ], ) def test_get_c128_boundary(starts, query_start_loc, expected): + query_start_loc_tensor = torch.tensor(query_start_loc) + query_lens = query_start_loc_tensor[1:] - query_start_loc_tensor[:-1] metadata = SimpleNamespace( - _num_computed_tokens_cpu=None if starts is None else torch.tensor(starts), - query_start_loc_cpu=torch.tensor(query_start_loc), + seq_lens_cpu_upper_bound=( + None if starts is None else torch.tensor(starts) + query_lens + ), + query_start_loc_cpu=query_start_loc_tensor, ) assert _get_c128_boundary(metadata) is expected diff --git a/tests/v1/attention/test_chunked_local_attention.py b/tests/v1/attention/test_chunked_local_attention.py index c2798c8f2f..0826b70730 100644 --- a/tests/v1/attention/test_chunked_local_attention.py +++ b/tests/v1/attention/test_chunked_local_attention.py @@ -181,7 +181,8 @@ def test_local_attention_virtual_batches(test_data: LocalAttentionTestData): # Convert to numpy for easier comparison actual_q_seqlens = np.diff(result.query_start_loc_cpu.numpy()) - actual_k_seqlens = result.seq_lens_cpu.numpy() + assert result.seq_lens_cpu_upper_bound is not None + actual_k_seqlens = result.seq_lens_cpu_upper_bound.numpy() # Check that all query lengths are less than or equal to attn_chunk_size assert all(q_len <= attn_chunk_size for q_len in actual_q_seqlens) diff --git a/tests/v1/attention/utils.py b/tests/v1/attention/utils.py index 590c9efb8a..48e618565c 100644 --- a/tests/v1/attention/utils.py +++ b/tests/v1/attention/utils.py @@ -85,13 +85,6 @@ def create_common_attn_metadata( seq_lens_cpu = seq_lens.cpu() max_seq_len = int(seq_lens_cpu.max()) - # Create computed tokens (context length for each sequence) - context_lens = [ - batch_spec.seq_lens[i] - batch_spec.query_lens[i] - for i in range(batch_spec.batch_size) - ] - num_computed_tokens_cpu = torch.tensor(context_lens, dtype=torch.int32) - # Create block table and slot mapping max_blocks = (max(batch_spec.seq_lens) + block_size - 1) // block_size if arange_block_indices: @@ -122,8 +115,6 @@ def create_common_attn_metadata( query_start_loc_cpu=query_start_loc_cpu, seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu, - _seq_lens_cpu=seq_lens_cpu, - _num_computed_tokens_cpu=num_computed_tokens_cpu, num_reqs=batch_spec.batch_size, num_actual_tokens=num_tokens, max_query_len=max_query_len, diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index c020a5db54..97b846219f 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -2386,8 +2386,8 @@ def test_retention_snapshots_handed_off_with_exact_block_ids(): def test_snapshot_handoff_dense_default_retention(): """With dense (default) retention, every materialized mamba boundary state block is handed off — regular mamba-align + prefix-match-unit - deployments get store-able boundary snapshots without setting - VLLM_PREFIX_CACHE_RETENTION_INTERVAL.""" + deployments get store-able boundary snapshots without setting a + prefix-cache retention interval.""" hash_block_size = 2 block_size = 4 manager = make_kv_cache_manager( diff --git a/tests/v1/e2e/spec_decode/draft_model/test_async.py b/tests/v1/e2e/spec_decode/draft_model/test_async.py index d7f932b0c2..ebb471ea71 100644 --- a/tests/v1/e2e/spec_decode/draft_model/test_async.py +++ b/tests/v1/e2e/spec_decode/draft_model/test_async.py @@ -1,70 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test that verifies no implicit GPU-CPU synchronization occurs during -speculative decoding generation under expected conditions. -""" - -import multiprocessing -import sys -import traceback +"""Tests for sync-free speculative decoding with async scheduling.""" import pytest -import torch - - -@pytest.fixture -def sync_tracker(): - """ - Fixture that patches CommonAttentionMetadata.seq_lens_cpu to detect - lazy init syncs. Prints stack traces immediately when syncs occur. - """ - from vllm.v1.attention.backend import CommonAttentionMetadata - - # Shared counter for cross-process communication (inherited by fork) - sync_count = multiprocessing.Value("i", 0) - - # Save original property - original_prop = CommonAttentionMetadata.seq_lens_cpu - original_fget = original_prop.fget - - # Create tracking wrapper - def tracking_seq_lens_cpu(self): - if self._seq_lens_cpu is None: - # Increment counter - with sync_count.get_lock(): - sync_count.value += 1 - count = sync_count.value - # Print stack trace immediately (shows in subprocess output) - print(f"\n{'=' * 60}", file=sys.stderr) - print(f"SYNC #{count}: seq_lens_cpu lazy init triggered!", file=sys.stderr) - print(f"{'=' * 60}", file=sys.stderr) - traceback.print_stack(file=sys.stderr) - print(f"{'=' * 60}\n", file=sys.stderr) - sys.stderr.flush() - return original_fget(self) - - # Apply patch - CommonAttentionMetadata.seq_lens_cpu = property(tracking_seq_lens_cpu) - - class SyncTracker: - @property - def count(self) -> int: - return sync_count.value - - def assert_no_sync(self, msg: str = ""): - count = sync_count.value - assert count == 0, ( - f"Unexpected GPU-CPU sync: seq_lens_cpu lazy init triggered " - f"{count} times. See stack traces above. {msg}" - ) - - yield SyncTracker() - - # Restore original property - CommonAttentionMetadata.seq_lens_cpu = original_prop - torch._dynamo.reset() - # Test configurations: (model, spec_model, method, num_spec_tokens, backend_env) SPEC_DECODE_CONFIGS = [ @@ -97,18 +35,16 @@ SPEC_DECODE_CONFIGS = [ SPEC_DECODE_CONFIGS, ) def test_no_sync_with_spec_decode( - sync_tracker, model: str, spec_model: str, method: str, num_spec_tokens: int, + monkeypatch, vllm_runner, ): - """ - Test that no implicit GPU-CPU sync occurs during speculative decoding - generation. - """ - # Import vLLM AFTER sync_tracker fixture has applied the patch + """Test generation while the worker rejects unintended GPU-CPU syncs.""" + monkeypatch.setenv("VLLM_GPU_SYNC_CHECK", "error") + from vllm import SamplingParams from vllm.config import CompilationConfig @@ -158,5 +94,3 @@ def test_no_sync_with_spec_decode( assert outputs[0].outputs[0].text, ( f"{method} target={model}: generated completion text is empty" ) - - sync_tracker.assert_no_sync() diff --git a/vllm/config/attention.py b/vllm/config/attention.py index d5a150c92a..799614a699 100644 --- a/vllm/config/attention.py +++ b/vllm/config/attention.py @@ -7,12 +7,9 @@ from typing import Any, Literal from pydantic import field_validator from vllm.config.utils import config -from vllm.logger import init_logger from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum from vllm.v1.attention.backends.registry import AttentionBackendEnum -logger = init_logger(__name__) - IndexerKVDType = Literal["auto", "bf16", "fp8", "mxfp4", "nvfp4"] MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"] @@ -64,10 +61,6 @@ class AttentionConfig: use_prefill_query_quantization: bool = False """If set, quantize query for attention in prefill.""" - use_fp4_indexer_cache: bool | None = None - """Deprecated alias for `indexer_kv_dtype`; use that instead. True maps to - `mxfp4`, False is a no-op (it selected the model default already).""" - indexer_kv_dtype: IndexerKVDType = "auto" """Data type for the sparse-attention indexer K cache. "auto" picks the model's default (bf16 for MiniMax M3, fp8 for the DeepSeek sparse @@ -116,20 +109,6 @@ class AttentionConfig: # layers still use the platform's normal automatic backend. self.backend = None - if self.use_fp4_indexer_cache is not None: - logger.warning( - "use_fp4_indexer_cache is deprecated and will be removed in " - "v0.29. Use indexer_kv_dtype instead (True -> 'mxfp4')." - ) - if self.use_fp4_indexer_cache: - if self.indexer_kv_dtype not in ("auto", "mxfp4"): - raise ValueError( - "use_fp4_indexer_cache=True conflicts with " - f"indexer_kv_dtype={self.indexer_kv_dtype!r}. Set only " - "indexer_kv_dtype." - ) - self.indexer_kv_dtype = "mxfp4" - def resolve_indexer_kv_dtype(self, default: IndexerKVDType) -> IndexerKVDType: """Resolve `indexer_kv_dtype`, substituting `default` for "auto".""" if self.indexer_kv_dtype == "auto": @@ -146,9 +125,7 @@ class AttentionConfig: """ from vllm.config.utils import get_hash_factors, hash_factors - # Folded into indexer_kv_dtype by __post_init__. - ignored_factors: set[str] = {"use_fp4_indexer_cache"} - factors = get_hash_factors(self, ignored_factors) + factors = get_hash_factors(self, set()) return hash_factors(factors) @field_validator("backend", mode="before") diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 93114ab1d1..9bd898731e 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar, Literal from pydantic import Field, field_validator, model_validator -from vllm.config.utils import config, get_from_deprecated_env_if_set +from vllm.config.utils import config from vllm.logger import init_logger from vllm.utils.torch_utils import ( is_quantized_kv_cache, @@ -58,15 +58,6 @@ CacheDType = Literal[ ] -def _get_prefix_cache_retention_interval() -> int | None: - env_value = get_from_deprecated_env_if_set( - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL", - "v0.29", - "prefix_cache_retention_interval", - ) - return 0 if env_value is None else int(env_value) - - MambaDType = Literal["auto", "float32", "float16", "bfloat16"] MambaCacheMode = Literal["all", "align", "none"] PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"] @@ -154,9 +145,7 @@ class CacheConfig: security risk tolerance against the performance benefits before turning this on. - "xxhash_cbor" combines canonical CBOR serialization with xxHash for reproducible hashing. Requires the optional ``xxhash`` package.""" - prefix_cache_retention_interval: int | None = Field( - default_factory=_get_prefix_cache_retention_interval, ge=0 - ) + prefix_cache_retention_interval: int | None = Field(default=0, ge=0) """Token interval between retained sliding-window and Mamba prefix-cache checkpoints. ``0`` retains only semantic checkpoints, including the latest replay boundary and shared-prefix junctions. Positive values additionally diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index ce2217d9fe..0d0dc22a27 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from pathlib import Path -from typing import Any, Literal, TypeAlias, TypedDict, cast, final +from typing import Any, Literal, TypeAlias, TypedDict, final import torch from pydantic import ConfigDict, Field, field_validator, model_validator @@ -11,7 +11,7 @@ from pydantic.dataclasses import dataclass import vllm.envs as envs from vllm.config.ec_transfer import ECTransferConfig -from vllm.config.utils import config, get_from_deprecated_env_if_set +from vllm.config.utils import config from vllm.logger import init_logger from vllm.utils.hashing import safe_hash from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -75,16 +75,6 @@ MMProcessorDevice: TypeAlias = str `"xpu"` on XPU). Validated against that set by the CLI.""" -def _get_mm_hasher_algorithm() -> MMHasherAlgorithm: - env_value = get_from_deprecated_env_if_set( - "VLLM_MM_HASHER_ALGORITHM", - "v0.27", - "mm_hasher_algorithm", - ) - env_value = "blake3" if env_value is None else env_value - return cast(MMHasherAlgorithm, env_value.lower()) - - MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions] """ A dictionary containing an entry for each modality type of dummy data. @@ -164,9 +154,7 @@ class MultiModalConfig: mm_processor_cache_type: MMCacheType = "lru" """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`, use shared memory FIFO cache. If `lru`, use mirrored LRU cache.""" - mm_hasher_algorithm: MMHasherAlgorithm = Field( - default_factory=_get_mm_hasher_algorithm - ) + mm_hasher_algorithm: MMHasherAlgorithm = "blake3" """Hash algorithm to use for multi-modal input caching. Use `"sha256"` or `"sha512"` for FIPS-compliant deployments.""" mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index a3cf1a5a1e..a3edb78750 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -305,7 +305,7 @@ class SchedulerOffloadConfig(NamedTuple): if retention_interval is not None: if retention_interval < 0: raise ValueError( - f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL " + f"prefix_cache_retention_interval " f"({retention_interval}) must be non-negative." ) for config in kv_group_configs: @@ -314,7 +314,7 @@ class SchedulerOffloadConfig(NamedTuple): and retention_interval % config.tokens_per_chunk != 0 ): raise ValueError( - f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL " + f"prefix_cache_retention_interval " f"({retention_interval}) must be a multiple of " f"tokens_per_chunk ({config.tokens_per_chunk})." ) diff --git a/vllm/envs.py b/vllm/envs.py index 18e04e3dc6..1c22466398 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -86,7 +86,6 @@ if TYPE_CHECKING: VLLM_MAX_IMAGE_PIXELS: int = 178_956_970 VLLM_VIDEO_LOADER_BACKEND: str = "opencv" VLLM_MEDIA_CONNECTOR: str = "http" - VLLM_MM_HASHER_ALGORITHM: str = "blake3" VLLM_TARGET_DEVICE: str = "cuda" VLLM_MAIN_CUDA_VERSION: str = "13.0" VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest" @@ -325,7 +324,6 @@ if TYPE_CHECKING: VLLM_LORA_ENABLE_DUAL_STREAM: bool = False VLLM_GPU_NIC_PCIE_MAPPING: str = "" VLLM_NIC_SELECTION_VARS: str = "" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None VLLM_ENABLE_HPC_OPS: bool = False @@ -1070,17 +1068,6 @@ environment_variables: dict[str, Callable[[], Any]] = { # imported at runtime. # If a non-existing backend is used, an AssertionError will be thrown. "VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"), - # Hash algorithm for multimodal content hashing. - # - "blake3": Default, fast cryptographic hash (not FIPS 140-3 compliant) - # - "sha256": FIPS 140-3 compliant, widely supported - # - "sha512": FIPS 140-3 compliant, faster on 64-bit systems - # Use sha256 or sha512 for FIPS compliance in government/enterprise deployments - "VLLM_MM_HASHER_ALGORITHM": env_with_choices( - "VLLM_MM_HASHER_ALGORITHM", - "blake3", - ["blake3", "sha256", "sha512"], - case_sensitive=False, - ), # Path to the XLA persistent cache directory. # Only used for XLA devices such as TPUs. "VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser( @@ -1172,11 +1159,6 @@ environment_variables: dict[str, Callable[[], Any]] = { if "VLLM_PLUGINS" not in os.environ else os.environ["VLLM_PLUGINS"].split(",") ), - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: ( - int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"]) - if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ - else None - ), # a local directory to look in for unrecognized LoRA adapters. # only works if plugins are enabled and # VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled. diff --git a/vllm/model_executor/layers/attention/cross_attention.py b/vllm/model_executor/layers/attention/cross_attention.py index 091f0a1856..9336bc17c2 100644 --- a/vllm/model_executor/layers/attention/cross_attention.py +++ b/vllm/model_executor/layers/attention/cross_attention.py @@ -115,7 +115,7 @@ def create_cross_attention_backend( # needed here to know how many tokens to attend to from the cached # cross-attention KV cache. new_metadata.seq_lens = common_attn_metadata.encoder_seq_lens - new_metadata._seq_lens_cpu = torch.from_numpy( + new_metadata.seq_lens_cpu_upper_bound = torch.from_numpy( common_attn_metadata.encoder_seq_lens_cpu ) diff --git a/vllm/model_executor/models/whisper_causal.py b/vllm/model_executor/models/whisper_causal.py index d76e9b0d4a..7004bba283 100644 --- a/vllm/model_executor/models/whisper_causal.py +++ b/vllm/model_executor/models/whisper_causal.py @@ -161,10 +161,8 @@ def create_whisper_attention_backend_with_block_pooling( new_common_attn_metadata.query_start_loc *= block_pool_size new_common_attn_metadata.query_start_loc_cpu *= block_pool_size new_common_attn_metadata.seq_lens *= block_pool_size - if new_common_attn_metadata._seq_lens_cpu is not None: - new_common_attn_metadata._seq_lens_cpu *= block_pool_size - if new_common_attn_metadata._num_computed_tokens_cpu is not None: - new_common_attn_metadata._num_computed_tokens_cpu *= block_pool_size + if new_common_attn_metadata.seq_lens_cpu_upper_bound is not None: + new_common_attn_metadata.seq_lens_cpu_upper_bound *= block_pool_size new_common_attn_metadata.num_actual_tokens *= block_pool_size new_common_attn_metadata.max_query_len *= block_pool_size new_common_attn_metadata.max_seq_len *= block_pool_size diff --git a/vllm/models/deepseek_v4/compressor.py b/vllm/models/deepseek_v4/compressor.py index 3bc636945a..4da343b2c9 100644 --- a/vllm/models/deepseek_v4/compressor.py +++ b/vllm/models/deepseek_v4/compressor.py @@ -43,10 +43,12 @@ def _prefer_two_stage_compressor() -> bool: def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None: - starts = metadata._num_computed_tokens_cpu - if starts is None: + seq_lens_cpu = metadata.seq_lens_cpu_upper_bound + if seq_lens_cpu is None: return None + query_lens = metadata.query_start_loc_cpu[1:] - metadata.query_start_loc_cpu[:-1] + starts = seq_lens_cpu - query_lens starts_list = starts.tolist() query_start_loc = metadata.query_start_loc_cpu.tolist() return any( @@ -116,6 +118,10 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder): _, _, num_decode_tokens, _ = split_decodes_and_prefills( common_attn_metadata, decode_threshold=1 ) + async_spec_decode = ( + self.vllm_config.scheduler_config.async_scheduling + and self.vllm_config.speculative_config is not None + ) return CompressorMetadata( block_table=common_attn_metadata.block_table_tensor.clamp_(min=0), slot_mapping=common_attn_metadata.slot_mapping, @@ -124,7 +130,7 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder): num_decode_tokens=num_decode_tokens, c128_boundary=( _get_c128_boundary(common_attn_metadata) - if self.block_size == 8 + if self.block_size == 8 and not async_spec_decode else None ), ) diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 297efc632f..9459399441 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -130,14 +130,6 @@ def _sync_hip_cuda_env_vars(): hip_val = os.environ.get("HIP_VISIBLE_DEVICES") or None cuda_val = os.environ.get("CUDA_VISIBLE_DEVICES") or None - if cuda_val is not None: - logger.warning_once( - "Using CUDA_VISIBLE_DEVICES on ROCm is deprecated and support " - "will be removed in vLLM v0.26.0. Please use HIP_VISIBLE_DEVICES " - "instead.", - scope="process", - ) - if hip_val is not None and cuda_val is not None: if hip_val != cuda_val: raise ValueError( @@ -148,8 +140,6 @@ def _sync_hip_cuda_env_vars(): ) elif hip_val is not None: os.environ["CUDA_VISIBLE_DEVICES"] = hip_val - elif cuda_val is not None: - os.environ["HIP_VISIBLE_DEVICES"] = cuda_val # Sync at import time - catches misconfigurations from process start. diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index fcc937fe8a..d75c79fe0f 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any, ClassVar, Generic, Protocol, TypeVar import numpy as np import torch -from typing_extensions import deprecated from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Dynamic64Sym, @@ -452,10 +451,6 @@ class CommonAttentionMetadata: at the current decode run's last full-state write. write_pos counts from here, so a preemption-resumed request re-anchors past the prompt boundary.""" - # WARNING: Deprecated fields. Will be removed in a future release (v0.15.0) - _seq_lens_cpu: torch.Tensor | None = None - _num_computed_tokens_cpu: torch.Tensor | None = None - _num_computed_tokens_cache: torch.Tensor | None = None _token_to_req_indices_cache: torch.Tensor | None = None @@ -469,36 +464,6 @@ class CommonAttentionMetadata: def replace(self, **kwargs) -> "CommonAttentionMetadata": return replace(self, **kwargs) - @property - @deprecated( - """ - Prefer using device seq_lens directly to avoid implicit H<>D sync. - If a CPU copy is needed, use `seq_lens.cpu()` instead. - Will be removed in a future release, please migrate as soon as possible. - """ - ) - def seq_lens_cpu(self) -> torch.Tensor: - if self._seq_lens_cpu is None: - self._seq_lens_cpu = self.seq_lens.to("cpu") - return self._seq_lens_cpu - - @property - @deprecated( - """ - Prefer using device seq_lens directly to avoid implicit H<>D sync which breaks full - async scheduling. If a CPU copy is needed, it can be derived from - query_start_loc_cpu and seq_lens. - Will be removed in a future release, please migrate as soon as possible. - """ - ) - def num_computed_tokens_cpu(self) -> torch.Tensor: - if self._num_computed_tokens_cpu is None: - query_seq_lens = ( - self.query_start_loc_cpu[1:] - self.query_start_loc_cpu[:-1] - ) - self._num_computed_tokens_cpu = self.seq_lens_cpu - query_seq_lens - return self._num_computed_tokens_cpu - def compute_num_computed_tokens(self) -> torch.Tensor: """Compute num_computed_tokens on device (seq_lens - query_lens).""" if self._num_computed_tokens_cache is None: @@ -542,12 +507,6 @@ class CommonAttentionMetadata: query_start_loc=self.query_start_loc[: num_actual_reqs + 1], query_start_loc_cpu=self.query_start_loc_cpu[: num_actual_reqs + 1], seq_lens=self.seq_lens[:num_actual_reqs], - _seq_lens_cpu=self._seq_lens_cpu[:num_actual_reqs] - if self._seq_lens_cpu is not None - else None, - _num_computed_tokens_cpu=self._num_computed_tokens_cpu[:num_actual_reqs] - if self._num_computed_tokens_cpu is not None - else None, num_reqs=num_actual_reqs, num_actual_tokens=num_actual_tokens, max_query_len=self.max_query_len, @@ -564,6 +523,7 @@ class CommonAttentionMetadata: encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu), dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens), dcp_local_seq_lens_cpu=maybe_slice_reqs(self.dcp_local_seq_lens_cpu), + seq_lens_cpu_upper_bound=maybe_slice_reqs(self.seq_lens_cpu_upper_bound), is_prefilling=maybe_slice_reqs(self.is_prefilling), rswa_prefix_lens=maybe_slice_reqs(self.rswa_prefix_lens), replayssm_decode_base_cpu=maybe_slice_reqs(self.replayssm_decode_base_cpu), diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 35e9db46a5..0da68e3e3d 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -1421,7 +1421,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm if needs_seq_lens_cpu: with gpu_sync_allowed(): - seq_lens_cpu = common_attn_metadata.seq_lens_cpu + seq_lens_cpu = common_attn_metadata.seq_lens.cpu() else: seq_lens_cpu = None diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 1b9ee263db..e0b5fab465 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -633,13 +633,21 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0: decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu - num_computed_tokens_cpu = common_attn_metadata._num_computed_tokens_cpu - if decode_base_cpu is None or num_computed_tokens_cpu is None: + seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound + async_spec_decode = ( + self.vllm_config.scheduler_config.async_scheduling + and self.vllm_config.speculative_config is not None + ) + if decode_base_cpu is None or seq_lens_cpu is None or async_spec_decode: raise ValueError( - "--use-replayssm requires CPU decode-base and " - "computed-token counts to derive decode write positions" + "--use-replayssm requires exact CPU sequence lengths and " + "decode-base counts to derive decode write positions" ) - num_computed_d = num_computed_tokens_cpu[:num_decodes] + query_lens_cpu = ( + common_attn_metadata.query_start_loc_cpu[1 : num_decodes + 1] + - common_attn_metadata.query_start_loc_cpu[:num_decodes] + ) + num_computed_d = seq_lens_cpu[:num_decodes] - query_lens_cpu decode_base_d = decode_base_cpu[:num_decodes] align_mode = self.vllm_config.cache_config.mamba_cache_mode == "align" block_size = self.kv_cache_spec.block_size @@ -656,10 +664,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): # write_pos counts decode steps since the ring's last full-state # write (the anchor), so a resumed request re-anchors correctly. decode_steps_cpu = num_computed_d - effective_base - query_lens_cpu = ( - common_attn_metadata.query_start_loc_cpu[1 : num_decodes + 1] - - common_attn_metadata.query_start_loc_cpu[:num_decodes] - ) valid_decode_rows = query_lens_cpu > 0 # A single-token prefill row replayed as decode (query_len==1 with # prior state) has decode_steps < 0; force it to a one-token flush diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index c20eaab0e5..833e0b21d8 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -695,7 +695,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): if self.vllm_config.speculative_config else 0 ) - self.use_fp4_indexer_cache = dsa_indexer_uses_fp4(self.vllm_config) + self.indexer_uses_fp4 = dsa_indexer_uses_fp4(self.vllm_config) next_n = self.num_speculative_tokens + 1 self.decode_threshold = next_n @@ -708,7 +708,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.use_flattening, self.supports_varlen, next_n, - self.use_fp4_indexer_cache, + self.indexer_uses_fp4, ) sm_count = num_compute_units(self.device.index) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 45a413aab3..5fba8f4237 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -17,6 +17,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import np_to_pinned_tensor from vllm.v1.attention.backend import ( AttentionBackend, @@ -638,12 +639,14 @@ class ROCMAiterMLASparseMetadataBuilder( reduce_partial_map = None if self._use_persistent_metadata and not use_triton_sparse: num_reqs = common_attn_metadata.num_reqs + with gpu_sync_allowed(): + seq_lens_cpu = common_attn_metadata.seq_lens[:num_reqs].cpu().numpy() clamped_seq_lens = np.minimum( - common_attn_metadata.seq_lens_cpu[:num_reqs].numpy(), + seq_lens_cpu, self.topk_tokens, ) clamped_context_lens = np.minimum( - common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - seg_lengths, + seq_lens_cpu - seg_lengths, self.topk_tokens, ) metadata_key = ( diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 99716ca911..bc127ccb8b 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -478,7 +478,7 @@ def make_local_attention_virtual_batches( query_start_loc_np = common_attn_metadata.query_start_loc_cpu.numpy() with gpu_sync_allowed(): # TODO see https://github.com/vllm-project/vllm/pull/31852 - seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy() + seq_lens_np = common_attn_metadata.seq_lens.cpu().numpy() block_table = common_attn_metadata.block_table_tensor device = common_attn_metadata.query_start_loc.device @@ -541,8 +541,6 @@ def make_local_attention_virtual_batches( # seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1] seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32) seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block - num_computed_tokens_local = seqlens_k_local - seqlens_q_local - k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - ( rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks) ) @@ -612,9 +610,7 @@ def make_local_attention_virtual_batches( block_table_tensor=block_table_local, slot_mapping=common_attn_metadata.slot_mapping, causal=True, - seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound, - _seq_lens_cpu=seq_lens_cpu, - _num_computed_tokens_cpu=torch.from_numpy(num_computed_tokens_local), + seq_lens_cpu_upper_bound=seq_lens_cpu, ), make_block_table @@ -692,8 +688,6 @@ def make_kv_sharing_fast_prefill_common_attn_metadata( slot_mapping=common_attn_metadata.slot_mapping, causal=True, seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound, - _seq_lens_cpu=common_attn_metadata._seq_lens_cpu, - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, ) return common_attn_metadata diff --git a/vllm/v1/spec_decode/dflash.py b/vllm/v1/spec_decode/dflash.py index 626dd36dea..42d8ef035c 100644 --- a/vllm/v1/spec_decode/dflash.py +++ b/vllm/v1/spec_decode/dflash.py @@ -192,8 +192,6 @@ class DFlashProposer(SpecDecodeBaseProposer): torch.from_numpy(self.token_arange_np[: batch_size + 1]).clone() * num_query_per_req ), - _seq_lens_cpu=None, - _num_computed_tokens_cpu=None, seq_lens_cpu_upper_bound=new_seq_lens_cpu_upper_bound, num_reqs=cad.num_reqs, num_actual_tokens=num_query_total, diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 4b078c2aea..2edbd27d6e 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -669,9 +669,6 @@ class SpecDecodeBaseProposer: # (i.e., not the first proposal). if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None: common_attn_metadata.seq_lens -= num_rejected_tokens_gpu - # Invalidate the CPU-side shadows to avoid H<>D sync. - common_attn_metadata._seq_lens_cpu = None - common_attn_metadata._num_computed_tokens_cpu = None block_size = self.block_size assert block_size > 0, "block_size has not been initialized." @@ -798,10 +795,6 @@ class SpecDecodeBaseProposer: self.max_model_len, ) - if common_attn_metadata._seq_lens_cpu is not None: - common_attn_metadata._seq_lens_cpu += 1 - if common_attn_metadata._num_computed_tokens_cpu is not None: - common_attn_metadata._num_computed_tokens_cpu += 1 if common_attn_metadata.seq_lens_cpu_upper_bound is not None: common_attn_metadata.seq_lens_cpu_upper_bound += 1 @@ -1134,8 +1127,6 @@ class SpecDecodeBaseProposer: query_start_loc=common_attn_metadata.query_start_loc, seq_lens=common_attn_metadata.seq_lens, query_start_loc_cpu=query_start_loc_cpu, - _seq_lens_cpu=common_attn_metadata._seq_lens_cpu, - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound, num_reqs=common_attn_metadata.num_reqs, num_actual_tokens=total_num_tokens, @@ -1245,8 +1236,6 @@ class SpecDecodeBaseProposer: query_start_loc=async_tensor_h2d(new_query_start_loc_cpu, device=device), seq_lens=async_tensor_h2d(new_seq_lens_cpu, device=device), query_start_loc_cpu=new_query_start_loc_cpu, - _seq_lens_cpu=new_seq_lens_cpu, - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, seq_lens_cpu_upper_bound=new_seq_lens_cpu, num_reqs=common_attn_metadata.num_reqs, num_actual_tokens=total_num_tokens, diff --git a/vllm/v1/spec_decode/step3p5.py b/vllm/v1/spec_decode/step3p5.py index 47821cc9b2..109d326356 100644 --- a/vllm/v1/spec_decode/step3p5.py +++ b/vllm/v1/spec_decode/step3p5.py @@ -386,8 +386,6 @@ class Step3p5MTPProposer(EagleProposer): if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None: common_attn_metadata.seq_lens -= num_rejected_tokens_gpu - common_attn_metadata._seq_lens_cpu = None - common_attn_metadata._num_computed_tokens_cpu = None block_size = self.block_size assert block_size > 0, "block_size has not been initialized." diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 2730416f76..a14d00bd58 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2377,8 +2377,7 @@ class GPUModelRunner( num_prompt_tokens_cpu = self.input_batch.num_prompt_tokens_cpu_tensor[ :num_reqs_padded ] - seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs_padded] - seq_lens_cpu_upper_bound = seq_lens_cpu + seq_lens_cpu_upper_bound = self.optimistic_seq_lens_cpu[:num_reqs_padded] # is_prefilling: True if request is still in prefill phase. # Used by mamba backends to distinguish actual decodes from @@ -2388,11 +2387,6 @@ class GPUModelRunner( # misclassify padding as prefill in CUDA graph mode. is_prefilling[num_reqs:] = False - if self.use_async_spec_decode: - # GPU tensors are authoritative in async mode. - seq_lens_cpu = None - num_computed_tokens_cpu = None - # Compute mm_prefix bidirectional ranges before building # attention metadata so builders handle them during build(). # By default, ranges exceeding sliding_window are skipped to prevent @@ -2466,8 +2460,6 @@ class GPUModelRunner( query_start_loc=self.query_start_loc.gpu[: num_reqs_padded + 1], query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1], seq_lens=self.seq_lens[:num_reqs_padded], - _seq_lens_cpu=seq_lens_cpu, - _num_computed_tokens_cpu=num_computed_tokens_cpu, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, replayssm_decode_base_cpu=replayssm_decode_base_cpu, num_reqs=num_reqs_padded, diff --git a/vllm/v1/worker/ubatch_utils.py b/vllm/v1/worker/ubatch_utils.py index f9612b1e7f..9743322c41 100644 --- a/vllm/v1/worker/ubatch_utils.py +++ b/vllm/v1/worker/ubatch_utils.py @@ -280,23 +280,11 @@ def _make_metadata_with_slice( query_start_loc[1:] -= tokens_skipped query_start_loc_cpu[1:] -= tokens_skipped seq_lens = attn_metadata.seq_lens[request_slice] - # Read raw fields to avoid triggering the deprecated D2H-syncing properties. - seq_lens_cpu = ( - attn_metadata._seq_lens_cpu[request_slice] - if attn_metadata._seq_lens_cpu is not None - else None - ) seq_lens_cpu_upper_bound = ( attn_metadata.seq_lens_cpu_upper_bound[request_slice] if attn_metadata.seq_lens_cpu_upper_bound is not None else None ) - num_computed_tokens_cpu = ( - attn_metadata._num_computed_tokens_cpu[request_slice] - if attn_metadata._num_computed_tokens_cpu is not None - else None - ) - if splits_last_request: # NOTE: We use start_locs (the original query_start_loc_cpu) to calculate # the tokens skipped because query_start_loc_cpu might have been modified @@ -309,9 +297,6 @@ def _make_metadata_with_slice( # (not cudagraph compatible) seq_lens = seq_lens.clone() seq_lens[-1] -= tokens_skipped - if seq_lens_cpu is not None: - seq_lens_cpu = seq_lens_cpu.clone() - seq_lens_cpu[-1] -= tokens_skipped if seq_lens_cpu_upper_bound is not None: seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound.clone() seq_lens_cpu_upper_bound[-1] -= tokens_skipped @@ -346,8 +331,6 @@ def _make_metadata_with_slice( block_table_tensor=block_table_tensor, slot_mapping=slot_mapping, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, - _seq_lens_cpu=seq_lens_cpu, - _num_computed_tokens_cpu=num_computed_tokens_cpu, )