[Bugfix] Universally align block table width to 128 tokens (#50302)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
+12
-1
@@ -36,6 +36,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -237,12 +238,22 @@ class AttentionGroup:
|
||||
if kernel_block_size is not None
|
||||
else self.kv_cache_spec
|
||||
)
|
||||
builder_cls = self.backend.get_builder_cls()
|
||||
builder_kwargs = {}
|
||||
if builder_cls.requires_block_table_width:
|
||||
max_num_blocks = self.kv_cache_spec.max_num_blocks_per_req(
|
||||
vllm_config, vllm_config.model_config.max_model_len
|
||||
)
|
||||
builder_kwargs["block_table_width"] = get_block_table_width(
|
||||
max_num_blocks, self.kv_cache_spec.block_size, kernel_block_size
|
||||
)
|
||||
self.metadata_builders = [
|
||||
self.backend.get_builder_cls()(
|
||||
builder_cls(
|
||||
kv_cache_spec_builder,
|
||||
self.layer_names,
|
||||
vllm_config,
|
||||
device,
|
||||
**builder_kwargs,
|
||||
)
|
||||
for _ in range(num_metadata_builders)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user