Add support for BLOOM (#331)

This commit is contained in:
Woosuk Kwon
2023-07-03 13:12:35 -07:00
committed by GitHub
parent d6fa1be3a8
commit e41f06702c
11 changed files with 479 additions and 18 deletions
+4 -2
View File
@@ -1,7 +1,7 @@
from typing import Dict, List, Tuple
import torch
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
from xformers.ops import AttentionBias
from vllm.sampling_params import SamplingParams
from vllm.sequence import SequenceData
@@ -38,7 +38,6 @@ class InputMetadata:
self.max_context_len = max_context_len
self.block_tables = block_tables
self.attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
self.num_prompts = len(prompt_lens)
self.num_prompt_tokens = sum(prompt_lens)
self.num_generation_tokens = context_lens.shape[0]
@@ -50,6 +49,9 @@ class InputMetadata:
assert block_tables.shape[0] == self.num_generation_tokens
assert context_lens.shape[0] == self.num_generation_tokens
# Set during the execution of the first attention op.
self.attn_bias: List[AttentionBias] = []
def __repr__(self) -> str:
# Print only useful metadata.
return (f'InputMetadata('