[Kernel][Model] logits_soft_cap for Gemma2 with flashinfer (#6051)
Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
@@ -102,6 +102,8 @@ class FlashInferMetadata(AttentionMetadata):
|
||||
# The data type of the paged kv cache
|
||||
data_type: torch.dtype = None
|
||||
device: torch.device = torch.device("cuda")
|
||||
# Only used by gemma2 model
|
||||
logits_soft_cap: Optional[float] = None
|
||||
|
||||
def __post_init__(self):
|
||||
# Refer to
|
||||
@@ -271,9 +273,11 @@ class FlashInferImpl(AttentionImpl):
|
||||
else:
|
||||
assert prefill_meta is not None
|
||||
assert prefill_meta.prefill_wrapper is not None
|
||||
output = prefill_meta.prefill_wrapper.forward(query,
|
||||
kv_cache,
|
||||
causal=True)
|
||||
output = prefill_meta.prefill_wrapper.forward(
|
||||
query,
|
||||
kv_cache,
|
||||
logits_soft_cap=attn_metadata.logits_soft_cap,
|
||||
causal=True)
|
||||
else:
|
||||
assert attn_metadata.decode_metadata is not None
|
||||
assert attn_metadata.decode_metadata.decode_wrapper is not None
|
||||
@@ -281,5 +285,5 @@ class FlashInferImpl(AttentionImpl):
|
||||
query,
|
||||
kv_cache,
|
||||
sm_scale=self.scale,
|
||||
)
|
||||
logits_soft_cap=attn_metadata.logits_soft_cap)
|
||||
return output.view(num_tokens, hidden_size)
|
||||
|
||||
@@ -77,9 +77,9 @@ def get_attn_backend(
|
||||
return IpexAttnBackend
|
||||
elif backend == _Backend.FLASHINFER:
|
||||
logger.info("Using Flashinfer backend.")
|
||||
logger.warning(("Flashinfer will be stuck on llma-2-7b,"
|
||||
" please avoid using Flashinfer as the"
|
||||
"backend when running on llma-2-7b."))
|
||||
logger.warning(("Flashinfer will be stuck on llama-2-7b,"
|
||||
" please avoid using Flashinfer as the "
|
||||
"backend when running on llama-2-7b."))
|
||||
from vllm.attention.backends.flashinfer import FlashInferBackend
|
||||
return FlashInferBackend
|
||||
elif backend == _Backend.PALLAS:
|
||||
|
||||
@@ -38,7 +38,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.sequence import IntermediateTensors, SamplerOutput
|
||||
from vllm.utils import print_warning_once
|
||||
|
||||
from .interfaces import SupportsLoRA
|
||||
|
||||
@@ -137,12 +136,6 @@ class Gemma2Attention(nn.Module):
|
||||
dtype=torch.get_default_dtype(),
|
||||
)
|
||||
|
||||
if self.config.attn_logit_softcapping is not None:
|
||||
print_warning_once(
|
||||
"Gemma 2 normally uses attention logit soft-capping; "
|
||||
"soft-capping is currently incompatible with the flash "
|
||||
"attention kernels, so vLLM removes it to enable speed and "
|
||||
"efficiency gains of flash attention.")
|
||||
# FIXME(woosuk): While Gemma 2 uses sliding window attention for every
|
||||
# odd layer, vLLM currently ignores it and uses global attention for
|
||||
# all layers.
|
||||
|
||||
@@ -15,7 +15,7 @@ try:
|
||||
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
|
||||
from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
|
||||
from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
|
||||
except ImportError:
|
||||
BatchDecodeWithPagedKVCacheWrapper = None
|
||||
CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
|
||||
@@ -683,6 +683,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
|
||||
logits_soft_cap = getattr(self.model_config.hf_config,
|
||||
'attn_logit_softcapping', None)
|
||||
if logits_soft_cap is not None and self.attn_backend.get_name(
|
||||
) != "flashinfer":
|
||||
raise ValueError("Please use Flashinfer backend for models with"
|
||||
"logits_soft_cap (i.e., Gemma-2)."
|
||||
" Otherwise, the output might be wrong."
|
||||
" Set Flashinfer backend by "
|
||||
"export VLLM_ATTENTION_BACKEND=FLASHINFER.")
|
||||
|
||||
if self.attn_backend.get_name() == "flashinfer":
|
||||
if len(paged_kv_indptr) > 0:
|
||||
paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
|
||||
@@ -700,7 +710,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
|
||||
kv_cache_dtype = get_kv_cache_torch_dtype(self.kv_cache_dtype,
|
||||
self.model_config.dtype)
|
||||
|
||||
attn_metadata = self.attn_backend.make_metadata(
|
||||
num_prefills=num_prefills,
|
||||
slot_mapping=slot_mapping_tensor,
|
||||
@@ -721,7 +730,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
query_start_loc=query_start_loc,
|
||||
device=self.device,
|
||||
data_type=kv_cache_dtype,
|
||||
use_cuda_graph=use_captured_graph)
|
||||
use_cuda_graph=use_captured_graph,
|
||||
logits_soft_cap=logits_soft_cap)
|
||||
|
||||
else:
|
||||
attn_metadata = self.attn_backend.make_metadata(
|
||||
@@ -1196,7 +1206,8 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
if model_input.attn_metadata.use_cuda_graph:
|
||||
batch_size = model_input.input_tokens.shape[0]
|
||||
model_input.attn_metadata.decode_wrapper = self.graph_runners[
|
||||
batch_size].flashinfer_decode_wrapper
|
||||
model_input.
|
||||
virtual_engine][batch_size].flashinfer_decode_wrapper
|
||||
else:
|
||||
model_input.attn_metadata.decode_wrapper = \
|
||||
self.flashinfer_decode_wrapper
|
||||
|
||||
Reference in New Issue
Block a user