[v1][attention] Support Hybrid Allocator + FlashInfer (#21412)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
Chen Zhang
2025-07-30 01:45:29 +00:00
committed by GitHub
parent 0e36abf993
commit 555e7225bc
16 changed files with 85 additions and 57 deletions
+8 -5
View File
@@ -2521,7 +2521,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
elapsed_time, cuda_graph_size / (1 << 30))
def _initialize_single_attn_backend(
self, kv_cache_spec: KVCacheSpec
self, kv_cache_spec: KVCacheSpec, layer_names: list[str]
) -> tuple[AttentionBackend, AttentionMetadataBuilder]:
if isinstance(kv_cache_spec, AttentionSpec):
attn_backend_i = get_attn_backend(
@@ -2551,6 +2551,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
attn_metadata_builder_i = attn_backend_i.get_builder_cls()(
kv_cache_spec,
layer_names,
self.vllm_config,
self.device,
)
@@ -2574,8 +2575,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
kv_cache_config.kv_cache_groups):
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
attn_backend_i, attn_metadata_builder_i = \
self._initialize_single_attn_backend(kv_cache_spec)
attn_backend_i, attn_metadata_builder_i = (
self._initialize_single_attn_backend(
kv_cache_spec, kv_cache_group_spec.layer_names))
self.attn_backends.append(attn_backend_i)
self.attn_metadata_builders.append(attn_metadata_builder_i)
@@ -2606,8 +2608,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
assert len(attn_specs) == len(attn_layers), \
"All or none of the layers are expected to be encoder-only"
attn_backend, attn_metadata_builder = \
self._initialize_single_attn_backend(attn_specs[0])
attn_backend, attn_metadata_builder = (
self._initialize_single_attn_backend(attn_specs[0],
attn_layers.keys()))
self.attn_backends.append(attn_backend)
self.attn_metadata_builders.append(attn_metadata_builder)
self.is_encoder_only_model = True