[compile] Allow strings in custom ops without regressing compilation times (#38123)
Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
@@ -25,6 +25,9 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
@@ -414,7 +417,9 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
`vllm.forward_context.get_forward_context().attn_metadata`.
|
||||
"""
|
||||
if self.calculate_kv_scales:
|
||||
torch.ops.vllm.maybe_calc_kv_scales(query, key, value, self.layer_name)
|
||||
torch.ops.vllm.maybe_calc_kv_scales(
|
||||
query, key, value, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
output_dtype = query.dtype
|
||||
if self.query_quant is not None:
|
||||
# quantizing with a simple torch operation enables
|
||||
@@ -466,6 +471,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
else:
|
||||
# Skip this if sharing KV cache with an earlier attention layer.
|
||||
encoded = _encode_layer_name(self.layer_name)
|
||||
if (
|
||||
not self.attn_backend.forward_includes_kv_cache_update
|
||||
and self.kv_sharing_target_layer_name is None
|
||||
@@ -473,14 +479,14 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
and value is not None
|
||||
):
|
||||
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
|
||||
key, value, self.layer_name
|
||||
key, value, encoded
|
||||
)
|
||||
torch.ops.vllm.unified_attention_with_output(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
output,
|
||||
self.layer_name,
|
||||
encoded,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return output.view(-1, hidden_size)
|
||||
@@ -553,8 +559,9 @@ def maybe_calc_kv_scales(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
|
||||
@@ -570,7 +577,7 @@ def maybe_calc_kv_scales_fake(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
@@ -622,12 +629,13 @@ def get_attention_context(
|
||||
def unified_kv_cache_update(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
if layer_slot_mapping is not None:
|
||||
assert hasattr(attn_layer.impl, "do_kv_cache_update"), (
|
||||
@@ -647,7 +655,7 @@ def unified_kv_cache_update(
|
||||
def unified_kv_cache_update_fake(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, device=key.device, dtype=key.dtype)
|
||||
|
||||
@@ -666,7 +674,7 @@ def unified_attention_with_output(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
kv_cache_dummy_dep: torch.Tensor | None = None,
|
||||
@@ -675,6 +683,7 @@ def unified_attention_with_output(
|
||||
# that ensures torch.compile preserves ordering between KV cache update and
|
||||
# attention forward.
|
||||
del kv_cache_dummy_dep
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
attn_metadata, self, kv_cache, _ = get_attention_context(layer_name)
|
||||
|
||||
self.impl.forward(
|
||||
@@ -695,7 +704,7 @@ def unified_attention_with_output_fake(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
kv_cache_dummy_dep: torch.Tensor | None = None,
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.distributed.kv_transfer import (
|
||||
has_kv_transfer_group,
|
||||
is_v1_kv_transfer_group,
|
||||
)
|
||||
from vllm.utils.torch_utils import _resolve_layer_name
|
||||
|
||||
|
||||
def maybe_transfer_kv_layer(func: Callable) -> Callable:
|
||||
@@ -38,7 +39,7 @@ def maybe_transfer_kv_layer(func: Callable) -> Callable:
|
||||
if not has_kv_transfer_group() or not is_v1_kv_transfer_group():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
layer_name: str = args[layer_name_index]
|
||||
layer_name = _resolve_layer_name(args[layer_name_index])
|
||||
|
||||
# Extract attention context (metadata, layer, kv_cache, layer_slot_mapping)
|
||||
attn_metadata, _, kv_cache, _ = get_attention_context(layer_name)
|
||||
|
||||
@@ -240,6 +240,9 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer, has_nvidia_artifactory
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
@@ -473,7 +476,12 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
output_shape: torch.Size | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.calculate_kv_scales:
|
||||
torch.ops.vllm.maybe_calc_kv_scales(q, kv_c_normed, k_pe, self.layer_name)
|
||||
torch.ops.vllm.maybe_calc_kv_scales(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
_encode_layer_name(self.layer_name),
|
||||
)
|
||||
|
||||
if self.use_direct_call:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
@@ -505,10 +513,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
return output
|
||||
else:
|
||||
encoded = _encode_layer_name(self.layer_name)
|
||||
kv_cache_dummy_dep = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
self.layer_name,
|
||||
encoded,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
@@ -518,7 +527,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output,
|
||||
self.layer_name,
|
||||
encoded,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return output
|
||||
@@ -900,7 +909,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
def unified_mla_kv_cache_update(
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
@@ -908,6 +917,7 @@ def unified_mla_kv_cache_update(
|
||||
Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context = get_forward_context()
|
||||
attn_layer = forward_context.no_compile_layers[layer_name]
|
||||
kv_cache = attn_layer.kv_cache
|
||||
@@ -939,7 +949,7 @@ def unified_mla_kv_cache_update(
|
||||
def unified_mla_kv_cache_update_fake(
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
@@ -959,7 +969,7 @@ def unified_mla_attention_with_output(
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
kv_cache_dummy_dep: torch.Tensor | None = None,
|
||||
@@ -968,6 +978,7 @@ def unified_mla_attention_with_output(
|
||||
# that ensures torch.compile preserves ordering between KV cache update and
|
||||
# attention forward.
|
||||
del kv_cache_dummy_dep
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
attn_metadata, layer, kv_cache, _ = get_attention_context(layer_name)
|
||||
layer.forward_impl(
|
||||
q,
|
||||
@@ -986,7 +997,7 @@ def unified_mla_attention_with_output_fake(
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
kv_cache_dummy_dep: torch.Tensor | None = None,
|
||||
|
||||
@@ -10,7 +10,12 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadata,
|
||||
@@ -170,7 +175,9 @@ class StaticSinkAttention(Attention, CustomOp):
|
||||
)
|
||||
if not self.sink_populated:
|
||||
self_kv_cache = self.kv_cache
|
||||
torch.ops.vllm.maybe_populate_sink(self_kv_cache, self.layer_name)
|
||||
torch.ops.vllm.maybe_populate_sink(
|
||||
self_kv_cache, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
|
||||
return super().forward(query, key, value, output_shape)
|
||||
|
||||
@@ -224,8 +231,9 @@ class StaticSinkAttention(Attention, CustomOp):
|
||||
|
||||
def maybe_populate_sink(
|
||||
self_kv_cache: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
if self.sink_populated or self_kv_cache.numel() == 0:
|
||||
@@ -235,7 +243,7 @@ def maybe_populate_sink(
|
||||
|
||||
def maybe_populate_sink_fake(
|
||||
self_kv_cache: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
@@ -32,15 +32,15 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
HAS_OPAQUE_TYPE,
|
||||
ModuleName,
|
||||
_USE_LAYERNAME,
|
||||
LayerName,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
|
||||
def get_layer_from_name(layer_name: str) -> torch.nn.Module:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
if not HAS_OPAQUE_TYPE and layer_name == "from_forward_context":
|
||||
if not _USE_LAYERNAME and layer_name == "from_forward_context":
|
||||
all_moe_layers = forward_context.all_moe_layers
|
||||
assert all_moe_layers is not None
|
||||
moe_layer_index = forward_context.moe_layer_index
|
||||
@@ -55,21 +55,21 @@ def get_layer_from_name(layer_name: str) -> torch.nn.Module:
|
||||
return forward_context.no_compile_layers[layer_name]
|
||||
|
||||
|
||||
# On torch >= 2.11, layer_name is a hoisted ModuleName opaque object;
|
||||
# On torch >= 2.11, layer_name is a hoisted LayerName opaque object;
|
||||
# on older versions it remains a plain str.
|
||||
if TYPE_CHECKING:
|
||||
from typing import TypeAlias
|
||||
|
||||
_layer_name_type: TypeAlias = str | ModuleName
|
||||
_layer_name_type: TypeAlias = str | LayerName
|
||||
else:
|
||||
_layer_name_type = ModuleName if HAS_OPAQUE_TYPE else str
|
||||
_layer_name_type = LayerName if _USE_LAYERNAME else str
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
def _resolve_layer_name(layer_name: str | ModuleName) -> str:
|
||||
def _resolve_layer_name(layer_name: str | LayerName) -> str:
|
||||
from torch._library.fake_class_registry import FakeScriptObject
|
||||
|
||||
if isinstance(layer_name, ModuleName):
|
||||
if isinstance(layer_name, LayerName):
|
||||
return layer_name.value
|
||||
elif isinstance(layer_name, FakeScriptObject):
|
||||
return layer_name.real_obj.value
|
||||
@@ -331,9 +331,9 @@ class MoERunnerBase(MoERunner):
|
||||
assert len(trunc_sizes) == 1
|
||||
return func(states, trunc_sizes[0])
|
||||
|
||||
def _encode_layer_name(self) -> str | ModuleName:
|
||||
if HAS_OPAQUE_TYPE:
|
||||
return ModuleName(self.layer_name)
|
||||
def _encode_layer_name(self) -> str | LayerName:
|
||||
if _USE_LAYERNAME:
|
||||
return LayerName(self.layer_name)
|
||||
# Can be unavailable or None in unittests
|
||||
if (
|
||||
is_forward_context_available()
|
||||
|
||||
@@ -56,7 +56,12 @@ from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
@@ -568,7 +573,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
b,
|
||||
a,
|
||||
core_attn_out,
|
||||
self.prefix,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
@@ -1084,13 +1089,14 @@ def gdn_attention_core(
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""
|
||||
Custom op for the core attention computation.
|
||||
Only handles the convolution + recurrent attention part.
|
||||
Input/output projections are handled outside this op.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward_core(
|
||||
@@ -1106,7 +1112,7 @@ def gdn_attention_core_fake(
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Fake implementation for torch.compile."""
|
||||
return
|
||||
|
||||
@@ -36,7 +36,12 @@ from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
|
||||
|
||||
|
||||
@@ -228,7 +233,7 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
torch.ops.vllm.mamba_mixer(
|
||||
hidden_states,
|
||||
output,
|
||||
self.prefix,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
@@ -515,8 +520,9 @@ def split_batch_to_prefill_and_decode(
|
||||
def mamba_mixer(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self.forward_impl(hidden_states=hidden_states, output=output)
|
||||
@@ -525,7 +531,7 @@ def mamba_mixer(
|
||||
def mamba_mixer_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
@@ -44,7 +44,12 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
|
||||
@@ -536,7 +541,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
torch.ops.vllm.mamba_mixer2(
|
||||
projected_states,
|
||||
ssm_output,
|
||||
self.prefix,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
# 4. gated MLP
|
||||
@@ -927,8 +932,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
def mamba_mixer2(
|
||||
projected_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self.conv_ssm_forward(projected_states=projected_states, output=output)
|
||||
@@ -937,7 +943,7 @@ def mamba_mixer2(
|
||||
def mamba_mixer2_fake(
|
||||
projected_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
@@ -11,7 +11,12 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_mqa_logits, fp8_paged_mqa_logits, has_deep_gemm
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
)
|
||||
@@ -30,7 +35,7 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
def sparse_attn_indexer(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: str,
|
||||
k_cache_prefix: LayerNameType,
|
||||
kv_cache: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -46,6 +51,7 @@ def sparse_attn_indexer(
|
||||
# careful! this will be None in dummy run
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
k_cache_prefix = _resolve_layer_name(k_cache_prefix)
|
||||
|
||||
# assert isinstance(attn_metadata, dict)
|
||||
if not isinstance(attn_metadata, dict):
|
||||
@@ -253,7 +259,7 @@ def sparse_attn_indexer(
|
||||
|
||||
def sparse_attn_indexer_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: str,
|
||||
k_cache_prefix: LayerNameType,
|
||||
kv_cache: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -342,7 +348,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
):
|
||||
return torch.ops.vllm.sparse_attn_indexer(
|
||||
hidden_states,
|
||||
self.k_cache.prefix,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_fp8,
|
||||
k,
|
||||
@@ -366,7 +372,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
self.k_cache.prefix,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_fp8,
|
||||
k,
|
||||
|
||||
Reference in New Issue
Block a user