[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
@@ -104,12 +103,13 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
@property
|
||||
def attn_vllm_config(self) -> VllmConfig:
|
||||
# The draft's attention differs from the target's in causality.
|
||||
config = copy.copy(super().attn_vllm_config)
|
||||
config.attention_config = replace(
|
||||
self.vllm_config.attention_config,
|
||||
use_non_causal=self.requires_non_causal,
|
||||
return replace(
|
||||
self.vllm_config,
|
||||
attention_config=replace(
|
||||
self.vllm_config.attention_config,
|
||||
use_non_causal=self.requires_non_causal,
|
||||
),
|
||||
)
|
||||
return config
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
wants_full = cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
|
||||
Reference in New Issue
Block a user