[ModelRunner V2] Support encoder-only attention (#49331)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
co-authored by
Taneem Ibrahim
parent
0934b26790
commit
fdaa0d9e59
@@ -459,6 +459,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend(
|
||||
self.kv_cache_config, self.vllm_config, self.device
|
||||
)
|
||||
attn_cg_support = attn_cg_support.narrow(
|
||||
*self.model_state.get_additional_cg_support()
|
||||
)
|
||||
self.block_tables = BlockTables(
|
||||
block_sizes=block_sizes,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
|
||||
Reference in New Issue
Block a user