[ModelRunnerV2][Hybrid model] Support kernel block size in hybrid model (#38831)
Signed-off-by: MengqingCao <cmq0113@163.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Mengqing Cao <cmq0113@163.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
381edde1b9
commit
1223732dda
@@ -393,10 +393,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
) + spec.num_speculative_blocks
|
||||
max_num_blocks_per_group.append(max_num_blocks)
|
||||
|
||||
(self.attn_backends, self.attn_groups, attn_cg_support, kernel_block_sizes) = (
|
||||
init_attn_backend(self.kv_cache_config, self.vllm_config, self.device)
|
||||
self.attn_groups, attn_cg_support, kernel_block_sizes = init_attn_backend(
|
||||
self.kv_cache_config, self.vllm_config, self.device
|
||||
)
|
||||
|
||||
self.block_tables = BlockTables(
|
||||
block_sizes=block_sizes,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -440,7 +439,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kv_caches,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_cache_config,
|
||||
self.attn_backends,
|
||||
self.attn_groups,
|
||||
self.device,
|
||||
self.cache_config.cache_dtype,
|
||||
kernel_block_sizes,
|
||||
|
||||
Reference in New Issue
Block a user