iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Deprecation] Deprecate items scheduled for 0.29 (#55353)

Browse Source 
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-09-11 08:17:02 -07:00
committed by GitHub
parent 1e1060f998
commit 5fe77aecfc
28 changed files with 62 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/attention_benchmarks/mla_runner.py
-6
View File
@@ -409,10 +409,6 @@ def _build_attention_metadata(
seq_lens_cpu = torch.tensor(kv_lens, dtype=torch.int32)
seq_lens_gpu = seq_lens_cpu.to(device)
# Build num_computed_tokens (context length for each request)
context_lens = [kv_len - q_len for q_len, kv_len in zip(q_lens, kv_lens)]
num_computed_tokens_cpu = torch.tensor(context_lens, dtype=torch.int32)
# Build block table
num_blocks_per_req = [(kv + block_size - 1) // block_size for kv in kv_lens]
max_num_blocks = max(num_blocks_per_req)
@@ -454,8 +450,6 @@ def _build_attention_metadata(
query_start_loc_cpu=q_start_cpu,
seq_lens=seq_lens_gpu,
seq_lens_cpu_upper_bound=seq_lens_cpu,
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
slot_mapping=slot_mapping,
block_table_tensor=block_table_gpu,
dcp_local_seq_lens=None,
tests/engine/test_arg_utils.py
-18
View File
@@ -561,24 +561,6 @@ def test_prefix_cache_default():
assert engine_args.prefix_cache_retention_interval == 64
def test_prefix_cache_retention_interval_from_deprecated_env(
monkeypatch, caplog, disable_log_dedup
):
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
engine_args = EngineArgs()
assert engine_args.prefix_cache_retention_interval == 64
assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text
assert "deprecated" in caplog.text
assert "prefix_cache_retention_interval" in caplog.text
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = parser.parse_args(["--prefix-cache-retention-interval", "32"])
engine_args = EngineArgs.from_cli_args(args)
assert engine_args.prefix_cache_retention_interval == 32
@pytest.mark.parametrize(
("arg", "expected", "option"),
[
tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py
+2 -2
View File
@@ -81,7 +81,7 @@ def _make_common_metadata():
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
_seq_lens_cpu=seq_lens,
seq_lens_cpu_upper_bound=seq_lens,
num_reqs=2,
num_actual_tokens=2,
max_query_len=1,
@@ -99,7 +99,7 @@ def _make_mixed_common_metadata():
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
_seq_lens_cpu=seq_lens,
seq_lens_cpu_upper_bound=seq_lens,
num_reqs=2,
num_actual_tokens=5,
max_query_len=4,
tests/kernels/test_compressor_kv_cache.py
+6 -2
View File
@@ -705,9 +705,13 @@ def test_gfx950_compressed_cache_canonicalizes_nonfinite(writer: str) -> None:
],
)
def test_get_c128_boundary(starts, query_start_loc, expected):
query_start_loc_tensor = torch.tensor(query_start_loc)
query_lens = query_start_loc_tensor[1:] - query_start_loc_tensor[:-1]
metadata = SimpleNamespace(
_num_computed_tokens_cpu=None if starts is None else torch.tensor(starts),
query_start_loc_cpu=torch.tensor(query_start_loc),
seq_lens_cpu_upper_bound=(
None if starts is None else torch.tensor(starts) + query_lens
),
query_start_loc_cpu=query_start_loc_tensor,
)
assert _get_c128_boundary(metadata) is expected
tests/v1/attention/test_chunked_local_attention.py
+2 -1
View File
@@ -181,7 +181,8 @@ def test_local_attention_virtual_batches(test_data: LocalAttentionTestData):
# Convert to numpy for easier comparison
actual_q_seqlens = np.diff(result.query_start_loc_cpu.numpy())
actual_k_seqlens = result.seq_lens_cpu.numpy()
assert result.seq_lens_cpu_upper_bound is not None
actual_k_seqlens = result.seq_lens_cpu_upper_bound.numpy()
# Check that all query lengths are less than or equal to attn_chunk_size
assert all(q_len <= attn_chunk_size for q_len in actual_q_seqlens)
tests/v1/attention/utils.py
-9
View File
@@ -85,13 +85,6 @@ def create_common_attn_metadata(
seq_lens_cpu = seq_lens.cpu()
max_seq_len = int(seq_lens_cpu.max())
# Create computed tokens (context length for each sequence)
context_lens = [
batch_spec.seq_lens[i] - batch_spec.query_lens[i]
for i in range(batch_spec.batch_size)
]
num_computed_tokens_cpu = torch.tensor(context_lens, dtype=torch.int32)
# Create block table and slot mapping
max_blocks = (max(batch_spec.seq_lens) + block_size - 1) // block_size
if arange_block_indices:
@@ -122,8 +115,6 @@ def create_common_attn_metadata(
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu,
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
num_reqs=batch_spec.batch_size,
num_actual_tokens=num_tokens,
max_query_len=max_query_len,
tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py
+2 -2
View File
@@ -2386,8 +2386,8 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():
def test_snapshot_handoff_dense_default_retention():
"""With dense (default) retention, every materialized mamba boundary
state block is handed off — regular mamba-align + prefix-match-unit
deployments get store-able boundary snapshots without setting
VLLM_PREFIX_CACHE_RETENTION_INTERVAL."""
deployments get store-able boundary snapshots without setting a
prefix-cache retention interval."""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
tests/v1/e2e/spec_decode/draft_model/test_async.py
+5 -71
View File
@@ -1,70 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test that verifies no implicit GPU-CPU synchronization occurs during
speculative decoding generation under expected conditions.
"""
import multiprocessing
import sys
import traceback
"""Tests for sync-free speculative decoding with async scheduling."""
import pytest
import torch
@pytest.fixture
def sync_tracker():
"""
Fixture that patches CommonAttentionMetadata.seq_lens_cpu to detect
lazy init syncs. Prints stack traces immediately when syncs occur.
"""
from vllm.v1.attention.backend import CommonAttentionMetadata
# Shared counter for cross-process communication (inherited by fork)
sync_count = multiprocessing.Value("i", 0)
# Save original property
original_prop = CommonAttentionMetadata.seq_lens_cpu
original_fget = original_prop.fget
# Create tracking wrapper
def tracking_seq_lens_cpu(self):
if self._seq_lens_cpu is None:
# Increment counter
with sync_count.get_lock():
sync_count.value += 1
count = sync_count.value
# Print stack trace immediately (shows in subprocess output)
print(f"\n{'=' * 60}", file=sys.stderr)
print(f"SYNC #{count}: seq_lens_cpu lazy init triggered!", file=sys.stderr)
print(f"{'=' * 60}", file=sys.stderr)
traceback.print_stack(file=sys.stderr)
print(f"{'=' * 60}\n", file=sys.stderr)
sys.stderr.flush()
return original_fget(self)
# Apply patch
CommonAttentionMetadata.seq_lens_cpu = property(tracking_seq_lens_cpu)
class SyncTracker:
@property
def count(self) -> int:
return sync_count.value
def assert_no_sync(self, msg: str = ""):
count = sync_count.value
assert count == 0, (
f"Unexpected GPU-CPU sync: seq_lens_cpu lazy init triggered "
f"{count} times. See stack traces above. {msg}"
)
yield SyncTracker()
# Restore original property
CommonAttentionMetadata.seq_lens_cpu = original_prop
torch._dynamo.reset()
# Test configurations: (model, spec_model, method, num_spec_tokens, backend_env)
SPEC_DECODE_CONFIGS = [
@@ -97,18 +35,16 @@ SPEC_DECODE_CONFIGS = [
SPEC_DECODE_CONFIGS,
)
def test_no_sync_with_spec_decode(
sync_tracker,
model: str,
spec_model: str,
method: str,
num_spec_tokens: int,
monkeypatch,
vllm_runner,
):
"""
Test that no implicit GPU-CPU sync occurs during speculative decoding
generation.
"""
# Import vLLM AFTER sync_tracker fixture has applied the patch
"""Test generation while the worker rejects unintended GPU-CPU syncs."""
monkeypatch.setenv("VLLM_GPU_SYNC_CHECK", "error")
from vllm import SamplingParams
from vllm.config import CompilationConfig
@@ -158,5 +94,3 @@ def test_no_sync_with_spec_decode(
assert outputs[0].outputs[0].text, (
f"{method} target={model}: generated completion text is empty"
)
sync_tracker.assert_no_sync()
vllm/config/attention.py
+1 -24
View File
@@ -7,12 +7,9 @@ from typing import Any, Literal
from pydantic import field_validator
from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
IndexerKVDType = Literal["auto", "bf16", "fp8", "mxfp4", "nvfp4"]
MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"]
@@ -64,10 +61,6 @@ class AttentionConfig:
use_prefill_query_quantization: bool = False
"""If set, quantize query for attention in prefill."""
use_fp4_indexer_cache: bool | None = None
"""Deprecated alias for `indexer_kv_dtype`; use that instead. True maps to
`mxfp4`, False is a no-op (it selected the model default already)."""
indexer_kv_dtype: IndexerKVDType = "auto"
"""Data type for the sparse-attention indexer K cache. "auto" picks the
model's default (bf16 for MiniMax M3, fp8 for the DeepSeek sparse
@@ -116,20 +109,6 @@ class AttentionConfig:
# layers still use the platform's normal automatic backend.
self.backend = None
if self.use_fp4_indexer_cache is not None:
logger.warning(
"use_fp4_indexer_cache is deprecated and will be removed in "
"v0.29. Use indexer_kv_dtype instead (True -> 'mxfp4')."
)
if self.use_fp4_indexer_cache:
if self.indexer_kv_dtype not in ("auto", "mxfp4"):
raise ValueError(
"use_fp4_indexer_cache=True conflicts with "
f"indexer_kv_dtype={self.indexer_kv_dtype!r}. Set only "
"indexer_kv_dtype."
)
self.indexer_kv_dtype = "mxfp4"
def resolve_indexer_kv_dtype(self, default: IndexerKVDType) -> IndexerKVDType:
"""Resolve `indexer_kv_dtype`, substituting `default` for "auto"."""
if self.indexer_kv_dtype == "auto":
@@ -146,9 +125,7 @@ class AttentionConfig:
"""
from vllm.config.utils import get_hash_factors, hash_factors
# Folded into indexer_kv_dtype by __post_init__.
ignored_factors: set[str] = {"use_fp4_indexer_cache"}
factors = get_hash_factors(self, ignored_factors)
factors = get_hash_factors(self, set())
return hash_factors(factors)
@field_validator("backend", mode="before")
vllm/config/cache.py
+2 -13
View File
@@ -8,7 +8,7 @@ from typing import Any, ClassVar, Literal
from pydantic import Field, field_validator, model_validator
from vllm.config.utils import config, get_from_deprecated_env_if_set
from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.utils.torch_utils import (
is_quantized_kv_cache,
@@ -58,15 +58,6 @@ CacheDType = Literal[
]
def _get_prefix_cache_retention_interval() -> int | None:
env_value = get_from_deprecated_env_if_set(
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL",
"v0.29",
"prefix_cache_retention_interval",
)
return 0 if env_value is None else int(env_value)
MambaDType = Literal["auto", "float32", "float16", "bfloat16"]
MambaCacheMode = Literal["all", "align", "none"]
PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]
@@ -154,9 +145,7 @@ class CacheConfig:
security risk tolerance against the performance benefits before turning this on.
- "xxhash_cbor" combines canonical CBOR serialization with xxHash for
reproducible hashing. Requires the optional ``xxhash`` package."""
prefix_cache_retention_interval: int | None = Field(
default_factory=_get_prefix_cache_retention_interval, ge=0
)
prefix_cache_retention_interval: int | None = Field(default=0, ge=0)
"""Token interval between retained sliding-window and Mamba prefix-cache
checkpoints. ``0`` retains only semantic checkpoints, including the latest
replay boundary and shared-prefix junctions. Positive values additionally
vllm/config/multimodal.py
+3 -15
View File
@@ -3,7 +3,7 @@
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
from typing import Any, Literal, TypeAlias, TypedDict, final
import torch
from pydantic import ConfigDict, Field, field_validator, model_validator
@@ -11,7 +11,7 @@ from pydantic.dataclasses import dataclass
import vllm.envs as envs
from vllm.config.ec_transfer import ECTransferConfig
from vllm.config.utils import config, get_from_deprecated_env_if_set
from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -75,16 +75,6 @@ MMProcessorDevice: TypeAlias = str
`"xpu"` on XPU). Validated against that set by the CLI."""
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
env_value = get_from_deprecated_env_if_set(
"VLLM_MM_HASHER_ALGORITHM",
"v0.27",
"mm_hasher_algorithm",
)
env_value = "blake3" if env_value is None else env_value
return cast(MMHasherAlgorithm, env_value.lower())
MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
"""
A dictionary containing an entry for each modality type of dummy data.
@@ -164,9 +154,7 @@ class MultiModalConfig:
mm_processor_cache_type: MMCacheType = "lru"
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""
mm_hasher_algorithm: MMHasherAlgorithm = Field(
default_factory=_get_mm_hasher_algorithm
)
mm_hasher_algorithm: MMHasherAlgorithm = "blake3"
"""Hash algorithm to use for multi-modal input caching. Use `"sha256"` or
`"sha512"` for FIPS-compliant deployments."""
mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)
vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
+2 -2
View File
@@ -305,7 +305,7 @@ class SchedulerOffloadConfig(NamedTuple):
if retention_interval is not None:
if retention_interval < 0:
raise ValueError(
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
f"prefix_cache_retention_interval "
f"({retention_interval}) must be non-negative."
)
for config in kv_group_configs:
@@ -314,7 +314,7 @@ class SchedulerOffloadConfig(NamedTuple):
and retention_interval % config.tokens_per_chunk != 0
):
raise ValueError(
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "
f"prefix_cache_retention_interval "
f"({retention_interval}) must be a multiple of "
f"tokens_per_chunk ({config.tokens_per_chunk})."
)
vllm/envs.py
-18
View File
@@ -86,7 +86,6 @@ if TYPE_CHECKING:
VLLM_MAX_IMAGE_PIXELS: int = 178_956_970
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
VLLM_MEDIA_CONNECTOR: str = "http"
VLLM_MM_HASHER_ALGORITHM: str = "blake3"
VLLM_TARGET_DEVICE: str = "cuda"
VLLM_MAIN_CUDA_VERSION: str = "13.0"
VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest"
@@ -325,7 +324,6 @@ if TYPE_CHECKING:
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
VLLM_NIC_SELECTION_VARS: str = ""
VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None
VLLM_ENABLE_HPC_OPS: bool = False
@@ -1070,17 +1068,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# imported at runtime.
# If a non-existing backend is used, an AssertionError will be thrown.
"VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),
# Hash algorithm for multimodal content hashing.
# - "blake3": Default, fast cryptographic hash (not FIPS 140-3 compliant)
# - "sha256": FIPS 140-3 compliant, widely supported
# - "sha512": FIPS 140-3 compliant, faster on 64-bit systems
# Use sha256 or sha512 for FIPS compliance in government/enterprise deployments
"VLLM_MM_HASHER_ALGORITHM": env_with_choices(
"VLLM_MM_HASHER_ALGORITHM",
"blake3",
["blake3", "sha256", "sha512"],
case_sensitive=False,
),
# Path to the XLA persistent cache directory.
# Only used for XLA devices such as TPUs.
"VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(
@@ -1172,11 +1159,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
if "VLLM_PLUGINS" not in os.environ
else os.environ["VLLM_PLUGINS"].split(",")
),
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: (
int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"])
if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ
else None
),
# a local directory to look in for unrecognized LoRA adapters.
# only works if plugins are enabled and
# VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled.
vllm/model_executor/layers/attention/cross_attention.py
+1 -1
View File
@@ -115,7 +115,7 @@ def create_cross_attention_backend(
# needed here to know how many tokens to attend to from the cached
# cross-attention KV cache.
new_metadata.seq_lens = common_attn_metadata.encoder_seq_lens
new_metadata._seq_lens_cpu = torch.from_numpy(
new_metadata.seq_lens_cpu_upper_bound = torch.from_numpy(
common_attn_metadata.encoder_seq_lens_cpu
)
vllm/model_executor/models/whisper_causal.py
+2 -4
View File
@@ -161,10 +161,8 @@ def create_whisper_attention_backend_with_block_pooling(
new_common_attn_metadata.query_start_loc *= block_pool_size
new_common_attn_metadata.query_start_loc_cpu *= block_pool_size
new_common_attn_metadata.seq_lens *= block_pool_size
if new_common_attn_metadata._seq_lens_cpu is not None:
new_common_attn_metadata._seq_lens_cpu *= block_pool_size
if new_common_attn_metadata._num_computed_tokens_cpu is not None:
new_common_attn_metadata._num_computed_tokens_cpu *= block_pool_size
if new_common_attn_metadata.seq_lens_cpu_upper_bound is not None:
new_common_attn_metadata.seq_lens_cpu_upper_bound *= block_pool_size
new_common_attn_metadata.num_actual_tokens *= block_pool_size
new_common_attn_metadata.max_query_len *= block_pool_size
new_common_attn_metadata.max_seq_len *= block_pool_size
vllm/models/deepseek_v4/compressor.py
+9 -3
View File
@@ -43,10 +43,12 @@ def _prefer_two_stage_compressor() -> bool:
def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None:
starts = metadata._num_computed_tokens_cpu
if starts is None:
seq_lens_cpu = metadata.seq_lens_cpu_upper_bound
if seq_lens_cpu is None:
return None
query_lens = metadata.query_start_loc_cpu[1:] - metadata.query_start_loc_cpu[:-1]
starts = seq_lens_cpu - query_lens
starts_list = starts.tolist()
query_start_loc = metadata.query_start_loc_cpu.tolist()
return any(
@@ -116,6 +118,10 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
_, _, num_decode_tokens, _ = split_decodes_and_prefills(
common_attn_metadata, decode_threshold=1
)
async_spec_decode = (
self.vllm_config.scheduler_config.async_scheduling
and self.vllm_config.speculative_config is not None
)
return CompressorMetadata(
block_table=common_attn_metadata.block_table_tensor.clamp_(min=0),
slot_mapping=common_attn_metadata.slot_mapping,
@@ -124,7 +130,7 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
num_decode_tokens=num_decode_tokens,
c128_boundary=(
_get_c128_boundary(common_attn_metadata)
if self.block_size == 8
if self.block_size == 8 and not async_spec_decode
else None
),
)
vllm/platforms/rocm.py
-10
View File
@@ -130,14 +130,6 @@ def _sync_hip_cuda_env_vars():
hip_val = os.environ.get("HIP_VISIBLE_DEVICES") or None
cuda_val = os.environ.get("CUDA_VISIBLE_DEVICES") or None
if cuda_val is not None:
logger.warning_once(
"Using CUDA_VISIBLE_DEVICES on ROCm is deprecated and support "
"will be removed in vLLM v0.26.0. Please use HIP_VISIBLE_DEVICES "
"instead.",
scope="process",
)
if hip_val is not None and cuda_val is not None:
if hip_val != cuda_val:
raise ValueError(
@@ -148,8 +140,6 @@ def _sync_hip_cuda_env_vars():
)
elif hip_val is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = hip_val
elif cuda_val is not None:
os.environ["HIP_VISIBLE_DEVICES"] = cuda_val
# Sync at import time - catches misconfigurations from process start.
vllm/v1/attention/backend.py
+1 -41
View File
@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any, ClassVar, Generic, Protocol, TypeVar
import numpy as np
import torch
from typing_extensions import deprecated
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic64Sym,
@@ -452,10 +451,6 @@ class CommonAttentionMetadata:
at the current decode run's last full-state write. write_pos counts from
here, so a preemption-resumed request re-anchors past the prompt boundary."""
# WARNING: Deprecated fields. Will be removed in a future release (v0.15.0)
_seq_lens_cpu: torch.Tensor | None = None
_num_computed_tokens_cpu: torch.Tensor | None = None
_num_computed_tokens_cache: torch.Tensor | None = None
_token_to_req_indices_cache: torch.Tensor | None = None
@@ -469,36 +464,6 @@ class CommonAttentionMetadata:
def replace(self, **kwargs) -> "CommonAttentionMetadata":
return replace(self, **kwargs)
@property
@deprecated(
"""
Prefer using device seq_lens directly to avoid implicit H<>D sync.
If a CPU copy is needed, use `seq_lens.cpu()` instead.
Will be removed in a future release, please migrate as soon as possible.
"""
)
def seq_lens_cpu(self) -> torch.Tensor:
if self._seq_lens_cpu is None:
self._seq_lens_cpu = self.seq_lens.to("cpu")
return self._seq_lens_cpu
@property
@deprecated(
"""
Prefer using device seq_lens directly to avoid implicit H<>D sync which breaks full
async scheduling. If a CPU copy is needed, it can be derived from
query_start_loc_cpu and seq_lens.
Will be removed in a future release, please migrate as soon as possible.
"""
)
def num_computed_tokens_cpu(self) -> torch.Tensor:
if self._num_computed_tokens_cpu is None:
query_seq_lens = (
self.query_start_loc_cpu[1:] - self.query_start_loc_cpu[:-1]
)
self._num_computed_tokens_cpu = self.seq_lens_cpu - query_seq_lens
return self._num_computed_tokens_cpu
def compute_num_computed_tokens(self) -> torch.Tensor:
"""Compute num_computed_tokens on device (seq_lens - query_lens)."""
if self._num_computed_tokens_cache is None:
@@ -542,12 +507,6 @@ class CommonAttentionMetadata:
query_start_loc=self.query_start_loc[: num_actual_reqs + 1],
query_start_loc_cpu=self.query_start_loc_cpu[: num_actual_reqs + 1],
seq_lens=self.seq_lens[:num_actual_reqs],
_seq_lens_cpu=self._seq_lens_cpu[:num_actual_reqs]
if self._seq_lens_cpu is not None
else None,
_num_computed_tokens_cpu=self._num_computed_tokens_cpu[:num_actual_reqs]
if self._num_computed_tokens_cpu is not None
else None,
num_reqs=num_actual_reqs,
num_actual_tokens=num_actual_tokens,
max_query_len=self.max_query_len,
@@ -564,6 +523,7 @@ class CommonAttentionMetadata:
encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu),
dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens),
dcp_local_seq_lens_cpu=maybe_slice_reqs(self.dcp_local_seq_lens_cpu),
seq_lens_cpu_upper_bound=maybe_slice_reqs(self.seq_lens_cpu_upper_bound),
is_prefilling=maybe_slice_reqs(self.is_prefilling),
rswa_prefix_lens=maybe_slice_reqs(self.rswa_prefix_lens),
replayssm_decode_base_cpu=maybe_slice_reqs(self.replayssm_decode_base_cpu),
vllm/v1/attention/backends/flashinfer.py
+1 -1
View File
@@ -1421,7 +1421,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm
if needs_seq_lens_cpu:
with gpu_sync_allowed():
seq_lens_cpu = common_attn_metadata.seq_lens_cpu
seq_lens_cpu = common_attn_metadata.seq_lens.cpu()
else:
seq_lens_cpu = None
vllm/v1/attention/backends/mamba_attn.py
+13 -9
View File
@@ -633,13 +633,21 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0:
decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu
num_computed_tokens_cpu = common_attn_metadata._num_computed_tokens_cpu
if decode_base_cpu is None or num_computed_tokens_cpu is None:
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
async_spec_decode = (
self.vllm_config.scheduler_config.async_scheduling
and self.vllm_config.speculative_config is not None
)
if decode_base_cpu is None or seq_lens_cpu is None or async_spec_decode:
raise ValueError(
"--use-replayssm requires CPU decode-base and "
"computed-token counts to derive decode write positions"
"--use-replayssm requires exact CPU sequence lengths and "
"decode-base counts to derive decode write positions"
)
num_computed_d = num_computed_tokens_cpu[:num_decodes]
query_lens_cpu = (
common_attn_metadata.query_start_loc_cpu[1 : num_decodes + 1]
- common_attn_metadata.query_start_loc_cpu[:num_decodes]
)
num_computed_d = seq_lens_cpu[:num_decodes] - query_lens_cpu
decode_base_d = decode_base_cpu[:num_decodes]
align_mode = self.vllm_config.cache_config.mamba_cache_mode == "align"
block_size = self.kv_cache_spec.block_size
@@ -656,10 +664,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
# write_pos counts decode steps since the ring's last full-state
# write (the anchor), so a resumed request re-anchors correctly.
decode_steps_cpu = num_computed_d - effective_base
query_lens_cpu = (
common_attn_metadata.query_start_loc_cpu[1 : num_decodes + 1]
- common_attn_metadata.query_start_loc_cpu[:num_decodes]
)
valid_decode_rows = query_lens_cpu > 0
# A single-token prefill row replayed as decode (query_len==1 with
# prior state) has decode_steps < 0; force it to a one-token flush
vllm/v1/attention/backends/mla/indexer.py
+2 -2
View File
@@ -695,7 +695,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
if self.vllm_config.speculative_config
else 0
)
self.use_fp4_indexer_cache = dsa_indexer_uses_fp4(self.vllm_config)
self.indexer_uses_fp4 = dsa_indexer_uses_fp4(self.vllm_config)
next_n = self.num_speculative_tokens + 1
self.decode_threshold = next_n
@@ -708,7 +708,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
self.use_flattening,
self.supports_varlen,
next_n,
self.use_fp4_indexer_cache,
self.indexer_uses_fp4,
)
sm_count = num_compute_units(self.device.index)
vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
+5 -2
View File
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import np_to_pinned_tensor
from vllm.v1.attention.backend import (
AttentionBackend,
@@ -638,12 +639,14 @@ class ROCMAiterMLASparseMetadataBuilder(
reduce_partial_map = None
if self._use_persistent_metadata and not use_triton_sparse:
num_reqs = common_attn_metadata.num_reqs
with gpu_sync_allowed():
seq_lens_cpu = common_attn_metadata.seq_lens[:num_reqs].cpu().numpy()
clamped_seq_lens = np.minimum(
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy(),
seq_lens_cpu,
self.topk_tokens,
)
clamped_context_lens = np.minimum(
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - seg_lengths,
seq_lens_cpu - seg_lengths,
self.topk_tokens,
)
metadata_key = (
vllm/v1/attention/backends/utils.py
+2 -8
View File
@@ -478,7 +478,7 @@ def make_local_attention_virtual_batches(
query_start_loc_np = common_attn_metadata.query_start_loc_cpu.numpy()
with gpu_sync_allowed():
# TODO see https://github.com/vllm-project/vllm/pull/31852
seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy()
seq_lens_np = common_attn_metadata.seq_lens.cpu().numpy()
block_table = common_attn_metadata.block_table_tensor
device = common_attn_metadata.query_start_loc.device
@@ -541,8 +541,6 @@ def make_local_attention_virtual_batches(
# seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1]
seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32)
seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block
num_computed_tokens_local = seqlens_k_local - seqlens_q_local
k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - (
rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks)
)
@@ -612,9 +610,7 @@ def make_local_attention_virtual_batches(
block_table_tensor=block_table_local,
slot_mapping=common_attn_metadata.slot_mapping,
causal=True,
seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound,
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=torch.from_numpy(num_computed_tokens_local),
seq_lens_cpu_upper_bound=seq_lens_cpu,
), make_block_table
@@ -692,8 +688,6 @@ def make_kv_sharing_fast_prefill_common_attn_metadata(
slot_mapping=common_attn_metadata.slot_mapping,
causal=True,
seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound,
_seq_lens_cpu=common_attn_metadata._seq_lens_cpu,
_num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,
)
return common_attn_metadata
vllm/v1/spec_decode/dflash.py
-2
View File
@@ -192,8 +192,6 @@ class DFlashProposer(SpecDecodeBaseProposer):
torch.from_numpy(self.token_arange_np[: batch_size + 1]).clone()
* num_query_per_req
),
_seq_lens_cpu=None,
_num_computed_tokens_cpu=None,
seq_lens_cpu_upper_bound=new_seq_lens_cpu_upper_bound,
num_reqs=cad.num_reqs,
num_actual_tokens=num_query_total,
vllm/v1/spec_decode/llm_base_proposer.py
-11
View File
@@ -669,9 +669,6 @@ class SpecDecodeBaseProposer:
# (i.e., not the first proposal).
if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None:
common_attn_metadata.seq_lens -= num_rejected_tokens_gpu
# Invalidate the CPU-side shadows to avoid H<>D sync.
common_attn_metadata._seq_lens_cpu = None
common_attn_metadata._num_computed_tokens_cpu = None
block_size = self.block_size
assert block_size > 0, "block_size has not been initialized."
@@ -798,10 +795,6 @@ class SpecDecodeBaseProposer:
self.max_model_len,
)
if common_attn_metadata._seq_lens_cpu is not None:
common_attn_metadata._seq_lens_cpu += 1
if common_attn_metadata._num_computed_tokens_cpu is not None:
common_attn_metadata._num_computed_tokens_cpu += 1
if common_attn_metadata.seq_lens_cpu_upper_bound is not None:
common_attn_metadata.seq_lens_cpu_upper_bound += 1
@@ -1134,8 +1127,6 @@ class SpecDecodeBaseProposer:
query_start_loc=common_attn_metadata.query_start_loc,
seq_lens=common_attn_metadata.seq_lens,
query_start_loc_cpu=query_start_loc_cpu,
_seq_lens_cpu=common_attn_metadata._seq_lens_cpu,
_num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,
seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound,
num_reqs=common_attn_metadata.num_reqs,
num_actual_tokens=total_num_tokens,
@@ -1245,8 +1236,6 @@ class SpecDecodeBaseProposer:
query_start_loc=async_tensor_h2d(new_query_start_loc_cpu, device=device),
seq_lens=async_tensor_h2d(new_seq_lens_cpu, device=device),
query_start_loc_cpu=new_query_start_loc_cpu,
_seq_lens_cpu=new_seq_lens_cpu,
_num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,
seq_lens_cpu_upper_bound=new_seq_lens_cpu,
num_reqs=common_attn_metadata.num_reqs,
num_actual_tokens=total_num_tokens,
vllm/v1/spec_decode/step3p5.py
-2
View File
@@ -386,8 +386,6 @@ class Step3p5MTPProposer(EagleProposer):
if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None:
common_attn_metadata.seq_lens -= num_rejected_tokens_gpu
common_attn_metadata._seq_lens_cpu = None
common_attn_metadata._num_computed_tokens_cpu = None
block_size = self.block_size
assert block_size > 0, "block_size has not been initialized."
vllm/v1/worker/gpu_model_runner.py
+1 -9
View File
@@ -2377,8 +2377,7 @@ class GPUModelRunner(
num_prompt_tokens_cpu = self.input_batch.num_prompt_tokens_cpu_tensor[
:num_reqs_padded
]
seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs_padded]
seq_lens_cpu_upper_bound = seq_lens_cpu
seq_lens_cpu_upper_bound = self.optimistic_seq_lens_cpu[:num_reqs_padded]
# is_prefilling: True if request is still in prefill phase.
# Used by mamba backends to distinguish actual decodes from
@@ -2388,11 +2387,6 @@ class GPUModelRunner(
# misclassify padding as prefill in CUDA graph mode.
is_prefilling[num_reqs:] = False
if self.use_async_spec_decode:
# GPU tensors are authoritative in async mode.
seq_lens_cpu = None
num_computed_tokens_cpu = None
# Compute mm_prefix bidirectional ranges before building
# attention metadata so builders handle them during build().
# By default, ranges exceeding sliding_window are skipped to prevent
@@ -2466,8 +2460,6 @@ class GPUModelRunner(
query_start_loc=self.query_start_loc.gpu[: num_reqs_padded + 1],
query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1],
seq_lens=self.seq_lens[:num_reqs_padded],
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
replayssm_decode_base_cpu=replayssm_decode_base_cpu,
num_reqs=num_reqs_padded,
vllm/v1/worker/ubatch_utils.py
-17
View File
@@ -280,23 +280,11 @@ def _make_metadata_with_slice(
query_start_loc[1:] -= tokens_skipped
query_start_loc_cpu[1:] -= tokens_skipped
seq_lens = attn_metadata.seq_lens[request_slice]
# Read raw fields to avoid triggering the deprecated D2H-syncing properties.
seq_lens_cpu = (
attn_metadata._seq_lens_cpu[request_slice]
if attn_metadata._seq_lens_cpu is not None
else None
)
seq_lens_cpu_upper_bound = (
attn_metadata.seq_lens_cpu_upper_bound[request_slice]
if attn_metadata.seq_lens_cpu_upper_bound is not None
else None
)
num_computed_tokens_cpu = (
attn_metadata._num_computed_tokens_cpu[request_slice]
if attn_metadata._num_computed_tokens_cpu is not None
else None
)
if splits_last_request:
# NOTE: We use start_locs (the original query_start_loc_cpu) to calculate
# the tokens skipped because query_start_loc_cpu might have been modified
@@ -309,9 +297,6 @@ def _make_metadata_with_slice(
# (not cudagraph compatible)
seq_lens = seq_lens.clone()
seq_lens[-1] -= tokens_skipped
if seq_lens_cpu is not None:
seq_lens_cpu = seq_lens_cpu.clone()
seq_lens_cpu[-1] -= tokens_skipped
if seq_lens_cpu_upper_bound is not None:
seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound.clone()
seq_lens_cpu_upper_bound[-1] -= tokens_skipped
@@ -346,8 +331,6 @@ def _make_metadata_with_slice(
block_table_tensor=block_table_tensor,
slot_mapping=slot_mapping,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
)