[Bugfix] Universally align block table width to 128 tokens (#50302)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -13,6 +13,7 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
|
||||
def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
@@ -48,11 +49,14 @@ def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_storage_block_
|
||||
compress_ratio=4,
|
||||
)
|
||||
vllm_config = create_vllm_config(max_model_len=1024)
|
||||
max_num_blocks = kv_cache_spec.max_num_blocks_per_req(vllm_config, 1024)
|
||||
block_table_width = get_block_table_width(max_num_blocks, kv_cache_spec.block_size)
|
||||
builder = DeepseekV32IndexerMetadataBuilder(
|
||||
kv_cache_spec=kv_cache_spec,
|
||||
layer_names=["dummy"],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
block_table_width=block_table_width,
|
||||
)
|
||||
|
||||
# Construct a single request where:
|
||||
|
||||
@@ -50,6 +50,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
BACKENDS_TO_TEST = [
|
||||
AttentionBackendEnum.CUTLASS_MLA,
|
||||
@@ -1495,16 +1496,9 @@ def test_backend_correctness(
|
||||
batch_spec, block_size, device
|
||||
)
|
||||
|
||||
# Pad block table to meet requirement:
|
||||
# block_num % (128 / block_size) == 0
|
||||
required_divisor = int(128 / block_size)
|
||||
current_block_num = common_attn_metadata.block_table_tensor.shape[1]
|
||||
if current_block_num % required_divisor != 0:
|
||||
# Pad to next multiple of required_divisor
|
||||
padded_block_num = (
|
||||
(current_block_num + required_divisor - 1) // required_divisor
|
||||
) * required_divisor
|
||||
padding_cols = padded_block_num - current_block_num
|
||||
padded_block_num = get_block_table_width(current_block_num, block_size)
|
||||
if padding_cols := padded_block_num - current_block_num:
|
||||
padding = torch.zeros(
|
||||
(common_attn_metadata.block_table_tensor.shape[0], padding_cols),
|
||||
dtype=torch.int32,
|
||||
|
||||
@@ -46,6 +46,11 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker.block_table import (
|
||||
MultiGroupBlockTable,
|
||||
SlotMappingMode,
|
||||
get_block_table_width,
|
||||
)
|
||||
from vllm.v1.worker.gpu.lora_utils import LoraState
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras
|
||||
@@ -1380,6 +1385,29 @@ def test_hybrid_block_table_initialization():
|
||||
)
|
||||
|
||||
|
||||
def test_get_block_table_width_aligns_to_128_tokens():
|
||||
assert get_block_table_width(1875, 64) == 1876
|
||||
|
||||
|
||||
def test_get_block_table_width_splits_virtual_blocks():
|
||||
assert get_block_table_width(235, 256, 64) == 940
|
||||
|
||||
|
||||
def test_mamba_state_table_width_is_not_aligned():
|
||||
block_tables = MultiGroupBlockTable(
|
||||
max_num_reqs=1,
|
||||
max_num_batched_tokens=1,
|
||||
pin_memory=False,
|
||||
device=torch.device("cpu"),
|
||||
block_sizes=[39664],
|
||||
kernel_block_sizes=[39664],
|
||||
max_num_blocks=[1],
|
||||
slot_mapping_modes=[SlotMappingMode.NONE],
|
||||
)
|
||||
|
||||
assert block_tables[0].max_num_blocks_per_req == 1
|
||||
|
||||
|
||||
def test_input_batch_with_kernel_block_sizes():
|
||||
"""Test InputBatch initialization with kernel_block_sizes parameter."""
|
||||
max_num_reqs = 10
|
||||
|
||||
Reference in New Issue
Block a user