[Attention]: Use KVCacheSpec for AttentionMetadataBuilder type hints (#50148)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -20,7 +20,6 @@ from vllm.v1.attention.backends.utils import (
|
|||||||
)
|
)
|
||||||
from vllm.v1.attention.selector import get_attn_backend
|
from vllm.v1.attention.selector import get_attn_backend
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import (
|
||||||
AttentionSpec,
|
|
||||||
ChunkedLocalAttentionSpec,
|
ChunkedLocalAttentionSpec,
|
||||||
KVCacheSpec,
|
KVCacheSpec,
|
||||||
get_kv_quant_mode,
|
get_kv_quant_mode,
|
||||||
@@ -42,7 +41,7 @@ def create_chunked_local_attention_backend(
|
|||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
cls: type["AttentionMetadataBuilder"],
|
cls: type["AttentionMetadataBuilder"],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: KVCacheSpec,
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
# Explicit override in case the underlying builder specialized this getter.
|
# Explicit override in case the underlying builder specialized this getter.
|
||||||
# @override omitted only because of mypy limitation due to type variable.
|
# @override omitted only because of mypy limitation due to type variable.
|
||||||
|
|||||||
@@ -1722,6 +1722,8 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
|||||||
understand this class
|
understand this class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
kv_cache_spec: AttentionSpec
|
||||||
|
|
||||||
# Defines the level of query length support for this backend.
|
# Defines the level of query length support for this backend.
|
||||||
# - SINGLE_ONLY: Only single-token queries (no spec decode support)
|
# - SINGLE_ONLY: Only single-token queries (no spec decode support)
|
||||||
# - UNIFORM: Supports uniform multi-token queries (spec decode with uniform lengths)
|
# - UNIFORM: Supports uniform multi-token queries (spec decode with uniform lengths)
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ if TYPE_CHECKING:
|
|||||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||||
from vllm.platforms.interface import DeviceCapability
|
from vllm.platforms.interface import DeviceCapability
|
||||||
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVQuantMode
|
from vllm.v1.kv_cache_interface import KVCacheSpec, KVQuantMode
|
||||||
|
|
||||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||||
|
|
||||||
@@ -635,7 +635,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
kv_cache_spec: "AttentionSpec",
|
kv_cache_spec: "KVCacheSpec",
|
||||||
layer_names: list[str],
|
layer_names: list[str],
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
@@ -649,7 +649,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
|
|||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
cls: type["AttentionMetadataBuilder"],
|
cls: type["AttentionMetadataBuilder"],
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
kv_cache_spec: "AttentionSpec",
|
kv_cache_spec: "KVCacheSpec",
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
"""Get the cudagraph support level of this builder class."""
|
"""Get the cudagraph support level of this builder class."""
|
||||||
return cls._cudagraph_support
|
return cls._cudagraph_support
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ from vllm.v1.attention.backend import (
|
|||||||
CommonAttentionMetadata,
|
CommonAttentionMetadata,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
|
||||||
from vllm.v1.worker.cp_utils import (
|
from vllm.v1.worker.cp_utils import (
|
||||||
run_split_fa2_dcp_context_attention,
|
run_split_fa2_dcp_context_attention,
|
||||||
should_skip_dcp_context_attention,
|
should_skip_dcp_context_attention,
|
||||||
@@ -360,7 +360,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
|||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
cls,
|
cls,
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
kv_cache_spec: "AttentionSpec",
|
kv_cache_spec: "KVCacheSpec",
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
return cls._cudagraph_support
|
return cls._cudagraph_support
|
||||||
|
|
||||||
|
|||||||
@@ -76,6 +76,7 @@ from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
|||||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import (
|
||||||
AttentionSpec,
|
AttentionSpec,
|
||||||
|
KVCacheSpec,
|
||||||
KVQuantMode,
|
KVQuantMode,
|
||||||
UniformTypeKVCacheSpecs,
|
UniformTypeKVCacheSpecs,
|
||||||
)
|
)
|
||||||
@@ -617,6 +618,7 @@ class FlashInferMetadata:
|
|||||||
|
|
||||||
|
|
||||||
class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||||
|
kv_cache_spec: AttentionSpec
|
||||||
reorder_batch_threshold: int = 1
|
reorder_batch_threshold: int = 1
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -893,7 +895,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
cls: type["FlashInferMetadataBuilder"],
|
cls: type["FlashInferMetadataBuilder"],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: KVCacheSpec,
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
"""Get the cudagraph support level for FlashInfer attention.
|
"""Get the cudagraph support level for FlashInfer attention.
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from vllm.v1.attention.backends.utils import (
|
|||||||
mamba_get_block_table_tensor,
|
mamba_get_block_table_tensor,
|
||||||
split_decodes_and_prefills,
|
split_decodes_and_prefills,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec, MambaSpec
|
from vllm.v1.kv_cache_interface import MambaSpec
|
||||||
|
|
||||||
|
|
||||||
class GDNAttentionBackend(AttentionBackend):
|
class GDNAttentionBackend(AttentionBackend):
|
||||||
@@ -80,18 +80,18 @@ class GDNAttentionMetadata:
|
|||||||
|
|
||||||
|
|
||||||
class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]):
|
class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]):
|
||||||
|
kv_cache_spec: MambaSpec
|
||||||
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
reorder_batch_threshold: int = 1
|
reorder_batch_threshold: int = 1
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: MambaSpec,
|
||||||
layer_names: list[str],
|
layer_names: list[str],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
):
|
):
|
||||||
assert isinstance(kv_cache_spec, MambaSpec)
|
|
||||||
self.vllm_config = vllm_config
|
self.vllm_config = vllm_config
|
||||||
self.compilation_config = vllm_config.compilation_config
|
self.compilation_config = vllm_config.compilation_config
|
||||||
self.speculative_config = vllm_config.speculative_config
|
self.speculative_config = vllm_config.speculative_config
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ from vllm.v1.attention.backends.utils import (
|
|||||||
KVCacheLayoutType,
|
KVCacheLayoutType,
|
||||||
split_decodes_and_prefills,
|
split_decodes_and_prefills,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -158,7 +158,7 @@ class HpcAttnMetadataBuilder(AttentionMetadataBuilder[HpcAttnMetadata]):
|
|||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
cls: type["HpcAttnMetadataBuilder"],
|
cls: type["HpcAttnMetadataBuilder"],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: KVCacheSpec,
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
spec_config = vllm_config.speculative_config
|
spec_config = vllm_config.speculative_config
|
||||||
if (
|
if (
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from vllm.v1.attention.backends.utils import (
|
|||||||
mamba_get_block_table_tensor,
|
mamba_get_block_table_tensor,
|
||||||
split_decodes_and_prefills,
|
split_decodes_and_prefills,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec, MambaSpec
|
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||||
|
|
||||||
|
|
||||||
class LinearAttentionBackend(AttentionBackend):
|
class LinearAttentionBackend(AttentionBackend):
|
||||||
@@ -46,19 +46,19 @@ class LinearAttentionMetadata:
|
|||||||
|
|
||||||
|
|
||||||
class LinearAttentionMetadataBuilder(AttentionMetadataBuilder[LinearAttentionMetadata]):
|
class LinearAttentionMetadataBuilder(AttentionMetadataBuilder[LinearAttentionMetadata]):
|
||||||
|
kv_cache_spec: MambaSpec
|
||||||
reorder_batch_threshold: int = 1
|
reorder_batch_threshold: int = 1
|
||||||
|
|
||||||
_cudagraph_support = AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
_cudagraph_support = AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: MambaSpec,
|
||||||
layer_names: list[str],
|
layer_names: list[str],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
):
|
):
|
||||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||||
assert isinstance(kv_cache_spec, MambaSpec)
|
|
||||||
|
|
||||||
def build(
|
def build(
|
||||||
self,
|
self,
|
||||||
@@ -120,13 +120,13 @@ class BailingLinearAttentionMetadataBuilder(LinearAttentionMetadataBuilder):
|
|||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
cls,
|
cls,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: KVCacheSpec,
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
return AttentionCGSupport.UNIFORM_BATCH
|
return AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: MambaSpec,
|
||||||
layer_names: list[str],
|
layer_names: list[str],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from vllm.v1.attention.backends.mamba_attn import (
|
|||||||
BaseMambaAttentionMetadata,
|
BaseMambaAttentionMetadata,
|
||||||
BaseMambaAttentionMetadataBuilder,
|
BaseMambaAttentionMetadataBuilder,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
from vllm.v1.kv_cache_interface import MambaSpec
|
||||||
|
|
||||||
|
|
||||||
def compute_varlen_chunk_metadata(
|
def compute_varlen_chunk_metadata(
|
||||||
@@ -118,7 +118,7 @@ class Mamba2AttentionMetadataBuilder(
|
|||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: MambaSpec,
|
||||||
layer_names: list[str],
|
layer_names: list[str],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from vllm.v1.attention.backends.utils import (
|
|||||||
mamba_get_block_table_tensor,
|
mamba_get_block_table_tensor,
|
||||||
split_decodes_and_prefills,
|
split_decodes_and_prefills,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec, MambaSpec
|
from vllm.v1.kv_cache_interface import MambaSpec
|
||||||
|
|
||||||
M = TypeVar("M", bound="BaseMambaAttentionMetadata")
|
M = TypeVar("M", bound="BaseMambaAttentionMetadata")
|
||||||
|
|
||||||
@@ -83,6 +83,7 @@ class BaseMambaAttentionMetadata:
|
|||||||
|
|
||||||
|
|
||||||
class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||||
|
kv_cache_spec: MambaSpec
|
||||||
metadata_cls: type[M]
|
metadata_cls: type[M]
|
||||||
reorder_batch_threshold: int = 1
|
reorder_batch_threshold: int = 1
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
@@ -92,7 +93,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
|||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: MambaSpec,
|
||||||
layer_names: list[str],
|
layer_names: list[str],
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
@@ -107,7 +108,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
|||||||
self.use_replayssm = vllm_config.cache_config.use_replayssm
|
self.use_replayssm = vllm_config.cache_config.use_replayssm
|
||||||
self.replayssm_buffer_len = vllm_config.cache_config.replayssm_buffer_len
|
self.replayssm_buffer_len = vllm_config.cache_config.replayssm_buffer_len
|
||||||
|
|
||||||
assert isinstance(kv_cache_spec, MambaSpec)
|
|
||||||
scheduler_config = vllm_config.scheduler_config
|
scheduler_config = vllm_config.scheduler_config
|
||||||
self.decode_cudagraph_max_bs: int = scheduler_config.max_num_seqs
|
self.decode_cudagraph_max_bs: int = scheduler_config.max_num_seqs
|
||||||
if self.compilation_config.max_cudagraph_capture_size is not None:
|
if self.compilation_config.max_cudagraph_capture_size is not None:
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from vllm.v1.attention.backends.utils import (
|
|||||||
get_dcp_local_seq_lens,
|
get_dcp_local_seq_lens,
|
||||||
split_decodes_and_prefills,
|
split_decodes_and_prefills,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec, MLAAttentionSpec
|
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec
|
||||||
from vllm.v1.worker.cp_utils import get_kv_cache_shard_count
|
from vllm.v1.worker.cp_utils import get_kv_cache_shard_count
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
@@ -463,7 +463,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
cls,
|
cls,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_spec: AttentionSpec,
|
kv_cache_spec: KVCacheSpec,
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
return AttentionCGSupport.UNIFORM_BATCH
|
return AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ from vllm.v1.attention.ops.triton_turboquant_decode import (
|
|||||||
triton_turboquant_decode_attention,
|
triton_turboquant_decode_attention,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.ops.triton_turboquant_store import triton_turboquant_store
|
from vllm.v1.attention.ops.triton_turboquant_store import triton_turboquant_store
|
||||||
|
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||||
from vllm.v1.worker.workspace import (
|
from vllm.v1.worker.workspace import (
|
||||||
current_workspace_manager,
|
current_workspace_manager,
|
||||||
is_workspace_manager_initialized,
|
is_workspace_manager_initialized,
|
||||||
@@ -198,6 +199,7 @@ class TurboQuantMetadata(AttentionMetadata):
|
|||||||
class TurboQuantMetadataBuilder(AttentionMetadataBuilder[TurboQuantMetadata]):
|
class TurboQuantMetadataBuilder(AttentionMetadataBuilder[TurboQuantMetadata]):
|
||||||
"""Builds TurboQuantMetadata from scheduler output."""
|
"""Builds TurboQuantMetadata from scheduler output."""
|
||||||
|
|
||||||
|
kv_cache_spec: AttentionSpec
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
|
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
|
||||||
|
|||||||
@@ -169,7 +169,7 @@ def init_attn_backend(
|
|||||||
# Check cudagraph support for the attention backend
|
# Check cudagraph support for the attention backend
|
||||||
cg_support = builder.get_cudagraph_support(
|
cg_support = builder.get_cudagraph_support(
|
||||||
vllm_config,
|
vllm_config,
|
||||||
cast(AttentionSpec, group.kv_cache_spec),
|
group.kv_cache_spec,
|
||||||
)
|
)
|
||||||
if cg_support.value < min_cg_support.value:
|
if cg_support.value < min_cg_support.value:
|
||||||
min_cg_support = cg_support
|
min_cg_support = cg_support
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import Any, cast
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -15,11 +15,7 @@ from vllm.v1.attention.backend import (
|
|||||||
CommonAttentionMetadata,
|
CommonAttentionMetadata,
|
||||||
)
|
)
|
||||||
from vllm.v1.core.sched.output import NewRequestData
|
from vllm.v1.core.sched.output import NewRequestData
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec, KVCacheConfig
|
||||||
AttentionSpec,
|
|
||||||
EncoderOnlyAttentionSpec,
|
|
||||||
KVCacheConfig,
|
|
||||||
)
|
|
||||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||||
@@ -152,7 +148,7 @@ class EncoderOnlyModelState(DefaultModelState):
|
|||||||
for group in self.encoder_attn_groups:
|
for group in self.encoder_attn_groups:
|
||||||
builder = group.get_metadata_builder(0)
|
builder = group.get_metadata_builder(0)
|
||||||
cg_support = builder.get_cudagraph_support(
|
cg_support = builder.get_cudagraph_support(
|
||||||
self.vllm_config, cast(AttentionSpec, group.kv_cache_spec)
|
self.vllm_config, group.kv_cache_spec
|
||||||
)
|
)
|
||||||
if cg_support.value < support.value:
|
if cg_support.value < support.value:
|
||||||
support = cg_support
|
support = cg_support
|
||||||
|
|||||||
Reference in New Issue
Block a user