[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -26,8 +26,8 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
@@ -2308,12 +2308,11 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
CircularBufferSpec(
|
||||
block_size=mamba_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=mamba_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -2366,12 +2365,11 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
CircularBufferSpec(
|
||||
block_size=kpool_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=kpool_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
|
||||
@@ -88,7 +88,7 @@ def test_packed_alignment_preserves_hot_pages_and_generic_stride():
|
||||
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
|
||||
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
|
||||
alignment = lcm(1152, hot.page_size_bytes)
|
||||
stride = _pool_bytes_per_block(groups)
|
||||
stride = _pool_bytes_per_block(_mock_vllm_config("BLHNC"), groups)
|
||||
assert stride == alignment
|
||||
layout = KVCacheLayout.BLHNC
|
||||
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
|
||||
@@ -254,7 +254,7 @@ class TestCSALinearGrouping:
|
||||
def test_every_group_fits_one_packed_block(self):
|
||||
config = _shared_layout_config()
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC)
|
||||
|
||||
pages = _pages(groups)
|
||||
for group in groups:
|
||||
@@ -418,7 +418,7 @@ class TestCSALinearGrouping:
|
||||
groups = _get_packed_kv_cache_groups(config, specs)
|
||||
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
|
||||
|
||||
assert _get_kv_cache_bytes_per_block(groups) == sum(
|
||||
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == sum(
|
||||
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
|
||||
)
|
||||
# That block holds every GDN state at once, so the repeat pattern alone
|
||||
@@ -465,7 +465,7 @@ class TestSlidingWindowBucketCap:
|
||||
pages = _pages(groups)
|
||||
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
|
||||
main_bytes = sum(pages[n] for n in main.layer_names)
|
||||
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
|
||||
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == main_bytes
|
||||
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
|
||||
per_group = main_bytes // pages["layers.0.swa"]
|
||||
assert len(swa_groups) == -(-43 // per_group)
|
||||
@@ -479,9 +479,9 @@ class TestSlidingWindowBucketCap:
|
||||
class TestDensePacking:
|
||||
def test_bytes_per_block_is_largest_group(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
|
||||
groups
|
||||
)
|
||||
assert _get_kv_cache_bytes_per_block(
|
||||
groups, KVCacheLayout.BLHNC
|
||||
) == _expected_bytes_per_block(groups)
|
||||
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("BLHNC"), groups, MEMORY
|
||||
@@ -554,6 +554,20 @@ class TestDensePacking:
|
||||
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
|
||||
assert mla_tensor.block_stride == block_stride
|
||||
|
||||
def test_layer_outer_pads_only_unaligned_pages(self):
|
||||
"""Layer-outer block strides are pages, so only an unaligned page pads."""
|
||||
mla_page = _mla(512).page_size_bytes
|
||||
idx = replace(_mla(128), block_stride_alignment=12288)
|
||||
groups = [_uniform_group({"mla.0": _mla(512), "idx.0": idx})]
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("LBNHC"), groups, MEMORY
|
||||
)
|
||||
mla_tensor, idx_tensor = config.kv_cache_tensors
|
||||
assert config.num_blocks == MEMORY // (mla_page + 12288)
|
||||
assert (mla_tensor.block_stride, idx_tensor.block_stride) == (mla_page, 12288)
|
||||
assert idx_tensor.offset == mla_page * config.num_blocks
|
||||
_bind(config, "LBNHC")
|
||||
|
||||
def test_overlaid_groups_alias_and_stay_isolated(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
# Overlay models resolve to a block-outer layout at backend selection (the
|
||||
@@ -562,7 +576,9 @@ class TestDensePacking:
|
||||
_mock_vllm_config("BLNHC"), groups, MEMORY
|
||||
)
|
||||
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
|
||||
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
|
||||
assert _pool_bytes_per_block(
|
||||
_mock_vllm_config("BLNHC"), groups
|
||||
) == _expected_bytes_per_block(groups)
|
||||
|
||||
views = _bind(config, "BLNHC")
|
||||
assert set(views) == set(g1) | set(g2)
|
||||
@@ -667,7 +683,10 @@ class TestCompressorRingGroup:
|
||||
assert len(ring_groups) == 1
|
||||
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
config,
|
||||
groups,
|
||||
available_memory=64
|
||||
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
|
||||
)
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
|
||||
manager = KVCacheManager(
|
||||
@@ -716,7 +735,10 @@ class TestSWABoundedReplayGrouping:
|
||||
for g in groups
|
||||
) == [(False, 128)] * 4 + [(True, 0)]
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
config,
|
||||
groups,
|
||||
available_memory=64
|
||||
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([kv_cache_config]),
|
||||
|
||||
@@ -10,7 +10,7 @@ from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -19,7 +19,6 @@ import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
@@ -69,7 +68,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
@@ -1942,13 +1940,12 @@ def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.cache_config.kv_cache_layout = None
|
||||
tail = KpoolTailSpec(
|
||||
tail = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
|
||||
@@ -2906,564 +2903,77 @@ def test_generate_scheduler_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def _glm5_like_kv_cache_spec(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> tuple[dict[str, KVCacheSpec], list[str]]:
|
||||
"""(mamba, mamba, mamba, MLA + indexer) * 11 plus a trailing mamba."""
|
||||
def test_packed_groups_glm5_like_hybrid():
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=8192))
|
||||
vllm_config.cache_config.kv_cache_layout = "BLHNC"
|
||||
kpool, block_size = 4, 1024
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
mamba_layers = []
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
name = f"layers.{i}.linear_attn"
|
||||
kv_cache_spec[name] = mamba_spec_factory()
|
||||
mamba_layers.append(name)
|
||||
return kv_cache_spec, mamba_layers
|
||||
|
||||
|
||||
def _glm5_like_kv_cache_spec_with_tail(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
"""Production kpool=4 proportions: (mamba*3, MLA + indexer + tail) * 11.
|
||||
|
||||
The tail's logical page (2 * kpool * 2*indexer_head_dim * bf16) must fit
|
||||
inside the indexer page it parasitizes (block_size//kpool * 132 B), which
|
||||
is why the fixture drops the tokens_per_state=16 shape used above.
|
||||
"""
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(mamba_spec_factory)
|
||||
for i in range(3, 45, 4):
|
||||
kpool = 4
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = replace(
|
||||
cast(MLAAttentionSpec, kv_cache_spec[f"layers.{i}.indexer"]),
|
||||
tokens_per_state=kpool,
|
||||
for i in range(11):
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
block_stride_alignment=32 * 576 * 2,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.tail"] = KpoolTailSpec(
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=kpool,
|
||||
block_stride_alignment=64 * 132,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.tail"] = CircularBufferSpec(
|
||||
block_size=kpool,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=kpool,
|
||||
)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
def _tensor_by_layer(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheTensor]:
|
||||
return {
|
||||
layer_name: tensor
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
for layer_name in tensor.layers
|
||||
}
|
||||
|
||||
|
||||
def _layer_offset(tensor: KVCacheTensor, layer_name: str) -> int:
|
||||
return tensor.offset + tensor.layers.index(layer_name) * tensor.layer_stride
|
||||
|
||||
|
||||
def test_get_kv_cache_config_balanced_mamba_hybrid():
|
||||
"""Hybrid slot sharing: mamba layers co-own the MLA slot tensors."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
|
||||
# Round-robin into G = ceil(34 / 11) = 4 groups so every mamba layer
|
||||
# gets an MLA slot.
|
||||
assert len(groups) == 5
|
||||
assert len(uniform_groups) == 1
|
||||
assert len(uniform_groups[0].layer_names) == 22
|
||||
assert uniform_groups[0].kv_cache_spec.get_max_layers_per_page_size() == 11
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 4].layer_names
|
||||
# Mamba pages are padded up to the MLA page.
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_padded == mla_page
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
# Every block id is charged the full per-block sum.
|
||||
attn_blocks = uniform_groups[0].kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
|
||||
available_memory = bytes_per_block * 100 + 1
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, available_memory
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
|
||||
# Every logical layer has a view into one backing allocation. Mamba views
|
||||
# alias their corresponding MLA slot by using the same byte offset.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_layer_names = [f"layers.{i}.attn" for i in range(3, 45, 4)]
|
||||
for i, mla_name in enumerate(mla_layer_names):
|
||||
mla_tensor = tensors[mla_name]
|
||||
assert mla_tensor.block_stride == mla_page
|
||||
for group in mamba_groups:
|
||||
if i < len(group.layer_names):
|
||||
assert tensors[group.layer_names[i]].offset == mla_tensor.offset
|
||||
for i in range(11):
|
||||
assert tensors[f"layers.{4 * i + 3}.indexer"].block_stride == idx_page
|
||||
|
||||
total_allocated = next(iter({t.size for t in kv_cache_config.kv_cache_tensors}))
|
||||
assert total_allocated == bytes_per_block * 100
|
||||
assert 0 <= available_memory - total_allocated < bytes_per_block
|
||||
|
||||
assert get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config, kv_cache_config
|
||||
) == pytest.approx(100 / blocks_per_request)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
"""The kpool tail parasitizes the indexer tensors instead of getting its
|
||||
own: sibling idx/tail tensors paired by layer order, zero standalone tail
|
||||
bytes, one shared block per request, and no prefix-caching leakage from
|
||||
the kpool-sized scratch group."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec = _glm5_like_kv_cache_spec_with_tail()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
tail_logical_page = kv_cache_spec["layers.3.tail"].page_size_bytes
|
||||
assert tail_logical_page < idx_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
tail_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and all(
|
||||
isinstance(spec, KpoolTailSpec)
|
||||
for spec in group.kv_cache_spec.kv_cache_specs.values()
|
||||
for i in range(34):
|
||||
kv_cache_spec[f"layers.{i}.kda"] = MambaSpec(
|
||||
shapes=((3, 6144), (16, 128, 128)),
|
||||
dtypes=(torch.bfloat16, torch.float32),
|
||||
block_size=block_size,
|
||||
)
|
||||
)
|
||||
# The tail never prefix-caches; its page is padded up to the indexer page
|
||||
# so the runner's strided view rides the indexer storage.
|
||||
assert not tail_group.kv_cache_spec.prefix_cacheable
|
||||
tail_inner = cast(
|
||||
KpoolTailSpec, tail_group.kv_cache_spec.kv_cache_specs["layers.3.tail"]
|
||||
)
|
||||
assert tail_inner.page_size_padded == idx_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Tail layers get logical views but no additional storage: each view aliases
|
||||
# its sibling indexer at the same offset in the shared allocation.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 67
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
for i in range(11):
|
||||
idx_name = f"layers.{4 * i + 3}.indexer"
|
||||
tail_name = f"layers.{4 * i + 3}.tail"
|
||||
assert tensors[idx_name].offset == tensors[tail_name].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
# The layout detector surfaces the sibling names in layer order for the
|
||||
# accounting and connector paths.
|
||||
layout = kv_cache_utils._glm5_next_tensor_layout(kv_cache_config.kv_cache_groups)
|
||||
assert layout is not None
|
||||
assert layout[3] == [f"layers.{4 * i + 3}.indexer" for i in range(11)]
|
||||
assert layout[6] == [f"layers.{4 * i + 3}.tail" for i in range(11)]
|
||||
|
||||
# Accounting charges exactly one extra shared block per request for the
|
||||
# tail, not a per-sequence allocation.
|
||||
attn_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and group is not tail_group
|
||||
)
|
||||
attn_blocks = attn_group.kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group + 1
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
|
||||
|
||||
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
|
||||
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
|
||||
prefix-cache hash granularity: participating groups alone decide it."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def align_mamba():
|
||||
return new_mamba_spec(mamba_cache_mode="align")
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(
|
||||
vllm_config, _glm5_like_kv_cache_spec_with_tail(align_mamba)
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
hash_vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
kv_transfer_config=object(),
|
||||
)
|
||||
# gcd(attn 1024, mamba 16) with the tail's 4 excluded; scheduler size is
|
||||
# the lcm including the tail (1024 % 4 == 0, so it coincides).
|
||||
assert kv_cache_utils.resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, hash_vllm_config
|
||||
) == (1024, 16)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible():
|
||||
"""Reject GLM-5.3-Flash layouts whose Mamba page exceeds the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
# (512, 1024) fp32 state = 2 MiB per page > 1.125 MiB MLA page.
|
||||
def big_mamba_spec():
|
||||
return new_mamba_spec(shapes=((512, 1024),), dtypes=(torch.float32,))
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(big_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert big_mamba_spec().page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(ValueError, match="does not fit the MLA page"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
|
||||
"""Use the generic layout error when no kpool indexer is present."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBHNC"
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec(
|
||||
shapes=((512, 1024),), dtypes=(torch.float32,)
|
||||
)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert kv_cache_spec["layers.0.linear_attn"].page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(NotImplementedError, match="page size"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
|
||||
disable slot sharing; the layout re-pads them to the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def prepadded_mamba_spec():
|
||||
return new_mamba_spec(page_size_padded=294_912)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(prepadded_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert prepadded_mamba_spec().page_size_bytes < mla_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
attn = [
|
||||
g
|
||||
for g in groups
|
||||
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and isinstance(g.kv_cache_spec.first_spec, MLAAttentionSpec)
|
||||
]
|
||||
assert len(groups) == 5
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
|
||||
"""Round-robin mamba grouping keeps practical PP splits balanced: every
|
||||
stage's largest projected mamba group slice fits its projected MLA
|
||||
layers, so per-stage slot tensors all have an MLA owner."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
global_groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
# PP=2 split at transformer layer 22/23.
|
||||
# Stage 0 of a PP=2 split at transformer layer 22/23: 5 MLA(+indexer)
|
||||
# layers, projected mamba groups [5, 5, 4, 4].
|
||||
worker_spec = {n: s for n, s in kv_cache_spec.items() if int(n.split(".")[1]) <= 22}
|
||||
groups = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
global_groups, worker_spec
|
||||
)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
rings = [
|
||||
g
|
||||
for g in groups
|
||||
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and isinstance(g.kv_cache_spec.first_spec, CircularBufferSpec)
|
||||
]
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [5, 5, 4, 4]
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 5 * mla_page + 5 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
# Every projected Mamba slot aliases an MLA owner.
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.0.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.1.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.2.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.4.linear_attn"].offset
|
||||
# The last slot only hosts the two groups with a 5th projected layer.
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.21.linear_attn"].offset
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.22.linear_attn"].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatch):
|
||||
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
|
||||
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
|
||||
grouping bumps to 5 groups so every stage's projection keeps sharing
|
||||
on instead of silently falling back on stage 0."""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=4),
|
||||
)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [g for g in groups if isinstance(g.kv_cache_spec, MambaSpec)]
|
||||
assert [len(g.layer_names) for g in mamba_groups] == [7, 7, 7, 7, 6]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 5].layer_names
|
||||
|
||||
for rank in range(4):
|
||||
start, end = get_pp_indices(45, rank, 4)
|
||||
worker_spec = {
|
||||
n: s
|
||||
for n, s in kv_cache_spec.items()
|
||||
if start <= int(n.split(".")[1]) < end
|
||||
}
|
||||
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
groups, worker_spec
|
||||
mamba = [
|
||||
g
|
||||
for g in groups
|
||||
if isinstance(
|
||||
getattr(g.kv_cache_spec, "first_spec", g.kv_cache_spec), MambaSpec
|
||||
)
|
||||
assert kv_cache_utils._glm5_next_tensor_layout(projected) is not None
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
|
||||
"""Reject PP stages whose Mamba layers have no MLA slot to share."""
|
||||
from vllm.config import ParallelConfig
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
monkeypatch.setenv("VLLM_PP_LAYER_PARTITION", "3,42")
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
with pytest.raises(ValueError, match="VLLM_PP_LAYER_PARTITION"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
|
||||
"""Hybrid slot sharing must take precedence over the experimental
|
||||
enable_cross_layers_blocks packed layout: generic packing would give the
|
||||
MLA slots a strided view, breaking the contiguous virtual split."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"enable_cross_layers_blocks": "true"},
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config, kv_transfer_config=kv_transfer_config
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert all(
|
||||
tensor.block_stride in (mla_page, idx_page)
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_no_indexer():
|
||||
"""Kimi-Linear-like: MLA without indexer layers, idx_stride == 0."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
# 20 mamba + 7 MLA layers -> G = ceil(20 / 7) = 3 groups of [7, 7, 6].
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert len(groups) == 4
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [7, 7, 6]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 7 * mla_page
|
||||
assert len(attn) == 1 and len(attn[0].layer_names) == 22
|
||||
assert len(rings) == 1 and len(rings[0].layer_names) == 11
|
||||
# 11 * (1.18 MB MLA + 34 KB indexer) per block fits 12 fp32-SSM KDA states.
|
||||
assert len(mamba) == 3
|
||||
assert sum(len(g.layer_names) for g in mamba) == 34
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Each group has one tensor descriptor and all groups alias the same slots.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 4
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_names = [f"layers.{i}.attn" for i in (*range(3, 27, 4), 26)]
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
mla_offset = _layer_offset(tensors[mla_name], mla_name)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
mamba_name = group.layer_names[index]
|
||||
assert _layer_offset(tensors[mamba_name], mamba_name) == mla_offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_capacity_after_scheduler_unwrap():
|
||||
"""max_concurrency must survive the scheduler-config unwrap.
|
||||
|
||||
Regression for the balanced-mamba hybrid layout:
|
||||
``generate_scheduler_kv_cache_config`` flattens the MLA
|
||||
``UniformTypeKVCacheSpecs`` group into a single ``MLAAttentionSpec``, so the
|
||||
scheduler config holds an MLA spec (page size A) next to ``MambaSpec``
|
||||
groups (page size B). ``EngineCore._initialize_kv_caches`` calls
|
||||
``get_kv_cache_capacity`` on that unwrapped config; the uniform-page-size
|
||||
path used to assert-fail (``assert len(page_sizes) == 1``) on this topology.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
|
||||
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
|
||||
# Confirm we are exercising the regression path: the MLA group is no longer
|
||||
# UniformTypeKVCacheSpecs, so sibling specs have differing page sizes.
|
||||
assert not any(
|
||||
isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
for g in scheduler_config.kv_cache_groups
|
||||
)
|
||||
|
||||
unwrapped_groups = scheduler_config.kv_cache_groups
|
||||
expected_max_mem = kv_cache_utils._max_memory_usage_bytes_from_groups(
|
||||
vllm_config, unwrapped_groups
|
||||
)
|
||||
expected_pool = kv_cache_utils._pool_bytes_per_block(unwrapped_groups)
|
||||
expected_blocks_per_request = (
|
||||
expected_max_mem + expected_pool - 1
|
||||
) // expected_pool
|
||||
|
||||
_, max_concurrency = get_kv_cache_capacity(vllm_config, scheduler_config)
|
||||
assert max_concurrency > 0
|
||||
assert max_concurrency == pytest.approx(
|
||||
scheduler_config.num_blocks / expected_blocks_per_request
|
||||
vllm_config, groups, available_memory=2 * 1024**3
|
||||
)
|
||||
strides = {t.block_stride for t in kv_cache_config.kv_cache_tensors}
|
||||
assert len(strides) == 1
|
||||
(block_stride,) = strides
|
||||
# Blocks sit a whole number of 32-row MLA pages and 64-state indexer pages
|
||||
# apart so both caches can be re-paged for their kernels.
|
||||
assert block_stride % (32 * 576 * 2) == 0
|
||||
assert block_stride % (64 * 132) == 0
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
|
||||
@@ -4483,6 +3993,7 @@ def test_kv_cache_reserves_null_block_for_max_model_len(use_override):
|
||||
block_size = 16
|
||||
max_model_len = 512 # needs 512 / 16 = 32 blocks
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# 32 blocks -> only 31 usable after the null block: one short -> reject.
|
||||
@@ -4528,6 +4039,7 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
block_size = 16
|
||||
max_model_len = 512 # needs 512 / 16 = 32 blocks
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# 32 blocks -> only 31 usable after the null block: one short -> reject.
|
||||
|
||||
@@ -20,7 +20,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CircularBufferManager,
|
||||
FullAttentionManager,
|
||||
KpoolTailManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
SlidingWindowManager,
|
||||
@@ -29,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -406,15 +404,15 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
|
||||
@pytest.mark.parametrize("record_for_zeroing", [True, False])
|
||||
def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
"""A transferred ring at index zero skips zeroing even with a local hit."""
|
||||
spec = KpoolTailSpec(
|
||||
"""The kpool tail ring is never zeroed, transferred or not: a pool only
|
||||
reads ring slots its own request wrote."""
|
||||
spec = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
manager = KpoolTailManager(
|
||||
manager = CircularBufferManager(
|
||||
spec,
|
||||
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
|
||||
enable_caching=True,
|
||||
@@ -428,9 +426,7 @@ def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
)
|
||||
blocks = manager.req_to_blocks["request"]
|
||||
assert len(blocks) == 1
|
||||
assert manager.take_new_block_ids() == (
|
||||
[blocks[0].block_id] if record_for_zeroing else []
|
||||
)
|
||||
assert manager.take_new_block_ids() == []
|
||||
assert manager.allocate_new_blocks("request", 20, 20) == []
|
||||
assert manager.take_new_block_ids() == []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user