[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
@@ -7,19 +7,61 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend
|
||||
from vllm.models.deepseek_v4_1.sparse_mla import (
|
||||
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
|
||||
)
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
CompressedSlotMappingKernel,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
BuildPrefillChunkMetadataKernel,
|
||||
DeepseekV4IndexerBackend,
|
||||
DeepseekV32IndexerMetadataBuilder,
|
||||
DeepseekV41IndexerBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
ConvertReqIndexToGlobalIndexKernel,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
MLAAttentionSpec,
|
||||
compute_layer_kv_cache_shape_bytes,
|
||||
)
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
|
||||
def test_indexer_shares_uncompressed_block_size_with_deepseek_v4_mla():
|
||||
"""Packed MLA/indexer groups must retain 64 compressed rows per page."""
|
||||
kernel_block_size = select_common_block_size(
|
||||
256, [DeepseekV4SparseMLABackend, DeepseekV4IndexerBackend]
|
||||
)
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=4,
|
||||
)
|
||||
assert compute_layer_kv_cache_shape_bytes(spec, 2, kernel_block_size) == (
|
||||
2,
|
||||
1,
|
||||
64,
|
||||
132,
|
||||
)
|
||||
|
||||
|
||||
def test_indexer_preserves_deepseek_v41_mla_block_size():
|
||||
"""V4.1 retains its smaller pages independently of V4's C4 indexer."""
|
||||
block_size = DeepseekV41SparseMLABackend.get_supported_kernel_block_sizes()[0]
|
||||
assert isinstance(block_size, int)
|
||||
assert (
|
||||
select_common_block_size(
|
||||
block_size, [DeepseekV41SparseMLABackend, DeepseekV41IndexerBackend]
|
||||
)
|
||||
== block_size
|
||||
)
|
||||
|
||||
|
||||
def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
|
||||
@@ -398,6 +398,56 @@ class TestCSALinearGrouping:
|
||||
assert len(gdn) == 10
|
||||
|
||||
|
||||
class TestSlidingWindowBucketCap:
|
||||
def test_sliding_window_bucket_is_capped_at_the_main_page(self):
|
||||
"""DeepSeek-V4.1 shape: 43 SlidingWindowMLASpec SWA caches beside an
|
||||
unbalanced 8-layer paged MLA bucket. Left whole, the SWA bucket would
|
||||
set the block width and the MLA group would fill under a third of
|
||||
every block; capped, it splits into groups no wider than the MLA
|
||||
page."""
|
||||
config = _shared_layout_config()
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for layer, ratio in ((2, 2), (8, 2), (14, 2), (20, 1)):
|
||||
specs[f"layers.{layer}.attn"] = MLAAttentionSpec(
|
||||
block_size=128,
|
||||
num_kv_heads=1,
|
||||
head_size=584,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=ratio,
|
||||
alignment=576,
|
||||
)
|
||||
specs[f"layers.{layer}.idx"] = MLAAttentionSpec(
|
||||
block_size=128,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=ratio,
|
||||
alignment=576,
|
||||
)
|
||||
for layer in range(43):
|
||||
specs[f"layers.{layer}.swa"] = SlidingWindowMLASpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=584,
|
||||
dtype=torch.uint8,
|
||||
sliding_window=128,
|
||||
alignment=576,
|
||||
)
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
pages = _pages(groups)
|
||||
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
|
||||
main_bytes = sum(pages[n] for n in main.layer_names)
|
||||
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
|
||||
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
|
||||
per_group = main_bytes // pages["layers.0.swa"]
|
||||
assert len(swa_groups) == -(-43 // per_group)
|
||||
for g in swa_groups:
|
||||
assert sum(pages[n] for n in g.layer_names) <= main_bytes
|
||||
assert sorted(n for g in swa_groups for n in g.layer_names) == sorted(
|
||||
f"layers.{i}.swa" for i in range(43)
|
||||
)
|
||||
|
||||
|
||||
class TestDensePacking:
|
||||
def test_bytes_per_block_is_largest_group(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
@@ -548,3 +598,55 @@ class TestDensePacking:
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
|
||||
class TestCompressorRingGroup:
|
||||
def test_ring_beside_paged_groups_keeps_the_kv_block_as_scheduler_block(self):
|
||||
"""DeepSeek-V4.1 compressor state: a CircularBufferSpec ring (capacity
|
||||
8) beside 128-token paged MLA groups and 64-token SWA groups. The ring
|
||||
claims one block per request, never hashes, and does not disturb the
|
||||
scheduler/hash block sizes the paged groups imply."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
specs: dict[str, KVCacheSpec] = {
|
||||
"layers.2.attn": MLAAttentionSpec(
|
||||
block_size=128,
|
||||
num_kv_heads=1,
|
||||
head_size=584,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=2,
|
||||
alignment=576,
|
||||
),
|
||||
"layers.2.attn.compressor.state_cache": CircularBufferSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=1,
|
||||
head_size=1024,
|
||||
head_size_v=0,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
}
|
||||
for layer in range(4):
|
||||
specs[f"layers.{layer}.attn.swa_cache"] = SlidingWindowMLASpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=584,
|
||||
dtype=torch.uint8,
|
||||
sliding_window=128,
|
||||
alignment=576,
|
||||
)
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
ring_groups = [g for g in groups if "state_cache" in g.layer_names[0]]
|
||||
assert len(ring_groups) == 1
|
||||
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
)
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([kv_cache_config]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=64,
|
||||
scheduler_block_size=128,
|
||||
)
|
||||
assert len(manager.coordinator.single_type_managers) == len(groups)
|
||||
|
||||
@@ -333,6 +333,57 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
|
||||
assert torch.equal(x, torch.full((4,), 15.0, device="cuda"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keyword", [False, True])
|
||||
def test_quantized_activation_replay_does_not_retain_tensors(
|
||||
cuda_capture_stream, keyword
|
||||
):
|
||||
"""Release the producer tensors while preserving both buffers on replay."""
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
BreakableCUDAGraphCapture,
|
||||
eager_break_during_capture,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
||||
|
||||
@eager_break_during_capture
|
||||
def consume(x, data_out, scale_out):
|
||||
data_out.copy_(x.data)
|
||||
scale_out.copy_(x.scale)
|
||||
|
||||
source = torch.ones((128, 128), device="cuda", dtype=torch.bfloat16)
|
||||
data_out = source.to(torch.float8_e4m3fn)
|
||||
scale_out = source[:, ::32].to(torch.uint8)
|
||||
output = torch.empty_like(source)
|
||||
cap = BreakableCUDAGraphCapture()
|
||||
with cap:
|
||||
data = source.to(torch.float8_e4m3fn)
|
||||
scale = source[:, ::32].to(torch.uint8)
|
||||
activation = QuantizedActivation(
|
||||
data, scale, source.dtype, source.shape, kMxfp8Dynamic
|
||||
)
|
||||
data_ref, scale_ref = weakref.ref(data), weakref.ref(scale)
|
||||
if keyword:
|
||||
consume(x=activation, data_out=data_out, scale_out=scale_out)
|
||||
else:
|
||||
consume(activation, data_out, scale_out)
|
||||
output.copy_(data_out)
|
||||
|
||||
del data, scale, activation
|
||||
gc.collect()
|
||||
assert data_ref() is None
|
||||
assert scale_ref() is None
|
||||
|
||||
for value in (1, 2, 3):
|
||||
source.fill_(value)
|
||||
cap.replay()
|
||||
cuda_capture_stream.synchronize()
|
||||
torch.testing.assert_close(output, source, rtol=0, atol=0)
|
||||
torch.testing.assert_close(scale_out, source[:, ::32].to(torch.uint8))
|
||||
|
||||
|
||||
def test_eager_attention_inside_multistream_overlap(cuda_capture_stream):
|
||||
"""Handle an eager attention break inside a multi-stream overlap region."""
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
@@ -798,3 +799,101 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
# The scratch group's namespace never enters the store.
|
||||
scratch_prefix = worker.token_dbs[2].key_for(hs[0]).rsplit("@", 1)[0]
|
||||
assert not any(key.startswith(scratch_prefix) for key in store._data)
|
||||
|
||||
|
||||
def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
|
||||
"""A per-request ring group (CircularBufferSpec, capacity as block_size)
|
||||
sits beside the paged group: worker setup tolerates its block size (the
|
||||
DeepSeek-V4.1 compressor ring is 8 rows), saving a request stores paged
|
||||
blocks only, and lookup hits on the paged group alone."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
ring = CircularBufferSpec(
|
||||
block_size=8, num_kv_heads=1, head_size=64, head_size_v=0, dtype=torch.uint8
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=4 * full.page_size_bytes,
|
||||
layers=["L0"],
|
||||
layer_stride=4 * full.page_size_bytes,
|
||||
block_stride=full.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=4 * ring.page_size_bytes,
|
||||
layers=["L1"],
|
||||
layer_stride=4 * ring.page_size_bytes,
|
||||
block_stride=ring.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], ring),
|
||||
],
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
store = _DictStore()
|
||||
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.num_kv_head = 8
|
||||
assert worker.token_dbs[1].hash_block_size == 8
|
||||
|
||||
raw_full = torch.zeros(4 * full.page_size_bytes, dtype=torch.int8)
|
||||
raw_ring = torch.zeros(4 * ring.page_size_bytes, dtype=torch.int8)
|
||||
kv_caches = {
|
||||
"L0": dense_kv_cache_views(raw_full, full, 4, 1, KVCacheLayout.LBNHC)[0],
|
||||
"L1": dense_kv_cache_views(raw_ring, ring, 4, 1, KVCacheLayout.LBNHC)[0],
|
||||
}
|
||||
|
||||
def _fake_thread_init(*args, **kwargs):
|
||||
for v in list(args) + list(kwargs.values()):
|
||||
if isinstance(v, threading.Event):
|
||||
v.set()
|
||||
m = MagicMock()
|
||||
m.start = lambda: None
|
||||
return m
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
mooncake_store_worker,
|
||||
"KVCacheStoreSendingThread",
|
||||
side_effect=_fake_thread_init,
|
||||
),
|
||||
patch.object(
|
||||
mooncake_store_worker,
|
||||
"KVCacheStoreRecvingThread",
|
||||
side_effect=_fake_thread_init,
|
||||
),
|
||||
):
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
send_thread = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_databases=worker.token_dbs,
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=worker.tp_rank,
|
||||
group_put_steps=worker._group_tp_replication_factors,
|
||||
kv_role=worker.kv_role,
|
||||
ready_event=threading.Event(),
|
||||
enable_kv_event=False,
|
||||
group_participates=[True, False],
|
||||
)
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
save_req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
# The ring group holds one block for the request's lifetime.
|
||||
block_ids=([1, 2, 3, 4], [1]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
store_job_id=1,
|
||||
)
|
||||
send_thread.add_request(save_req)
|
||||
send_thread._handle_request(send_thread.request_queue.get())
|
||||
worker.store = store
|
||||
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs).hit_length == 64
|
||||
assert not any("@group:1" in key for key in store._data)
|
||||
|
||||
@@ -1251,3 +1251,112 @@ def test_csa_linear_remote_ple_is_copied_whole():
|
||||
tp_ratio=-2,
|
||||
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
|
||||
)
|
||||
|
||||
|
||||
def _make_ring_worker():
|
||||
"""Paged MLA group plus a per-layer ring group, no Mamba: the shape of a
|
||||
model whose sliding-window KV lives in per-request rings."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
paged_specs = {
|
||||
f"paged.{index}": MLAAttentionSpec(
|
||||
block_size=4, num_kv_heads=1, head_size=64, dtype=torch.float16
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
ring_specs = {
|
||||
f"ring.{index}": CircularBufferSpec(
|
||||
block_size=6,
|
||||
num_kv_heads=1,
|
||||
head_size=32,
|
||||
head_size_v=0,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
|
||||
def group(specs):
|
||||
uniform = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert uniform is not None
|
||||
return KVCacheGroupSpec(list(specs), uniform)
|
||||
|
||||
# Both rings overlay the second paged region, so the scratch group must
|
||||
# address only that region while the paged group spans both.
|
||||
tensor_regions = (("paged.0",), ("paged.1", "ring.0", "ring.1"))
|
||||
region_size, page_size = 512, 256
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=len(tensor_regions) * region_size,
|
||||
layers=[layer_name],
|
||||
layer_stride=region_size,
|
||||
block_stride=page_size,
|
||||
offset=region_index * region_size,
|
||||
)
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
],
|
||||
kv_cache_groups=[group(paged_specs), group(ring_specs)],
|
||||
)
|
||||
|
||||
vllm_config = create_vllm_config(block_size=4)
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [4]
|
||||
fake_backend.get_name.return_value = "TEST_ATTN"
|
||||
fake_backend.full_cls_name.return_value = "test.AttentionBackend"
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", _RecordingNixl),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
|
||||
tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(2)]
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
layer_name: tensors[region_index]
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
}
|
||||
)
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_ring_scratch_without_mamba_registers_and_addresses_its_own_regions():
|
||||
worker = _make_ring_worker()
|
||||
|
||||
assert worker._ple_group_index is None
|
||||
assert worker._scratch_region_indices == [1]
|
||||
|
||||
desc_ids = worker._compute_desc_ids(
|
||||
block_ids=([1], [0]),
|
||||
dst_num_blocks=2,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
# Paged block 1 in both regions; ring block 0 only in its scratch region.
|
||||
assert sorted(desc_ids.tolist()) == sorted([0 * 2 + 1, 1 * 2 + 1, 1 * 2 + 0])
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector i
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -90,3 +91,59 @@ def test_disk_mode_rejects_non_positive_capacity() -> None:
|
||||
"disk_capacity_bytes": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _make_ring_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
|
||||
"""A paged full-attention group beside a per-request ring group."""
|
||||
full = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
ring = CircularBufferSpec(
|
||||
block_size=6, num_kv_heads=1, head_size=32, head_size_v=0, dtype=DTYPE
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=_BYTES_PER_BLOCK * num_blocks,
|
||||
layers=["attn"],
|
||||
layer_stride=_BYTES_PER_BLOCK * num_blocks,
|
||||
block_stride=_BYTES_PER_BLOCK,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ring.page_size_bytes * num_blocks,
|
||||
layers=["ring"],
|
||||
layer_stride=ring.page_size_bytes * num_blocks,
|
||||
block_stride=ring.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["attn"], full),
|
||||
KVCacheGroupSpec(["ring"], ring),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_ring_scratch_group_is_accepted_and_never_offloaded() -> None:
|
||||
"""The ring group's capacity is not a token granularity, it holds no
|
||||
hashed blocks, and it never participates in CPU hits."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="SimpleCPUOffloadConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"cpu_bytes_to_use": _BYTES_PER_BLOCK * 8},
|
||||
)
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config=vllm_config,
|
||||
role=KVConnectorRole.SCHEDULER,
|
||||
kv_cache_config=_make_ring_kv_cache_config(),
|
||||
)
|
||||
scheduler = connector.scheduler_manager
|
||||
assert scheduler.fa_gidx == 0
|
||||
cpu_groups = scheduler.cpu_kv_cache_config.kv_cache_groups
|
||||
assert [g.kv_cache_spec.prefix_cacheable for g in cpu_groups] == [True, False]
|
||||
# Only the paged group takes part in CPU-side hit lookup.
|
||||
attention_groups = scheduler.cpu_coordinator.attention_groups
|
||||
assert [g.group_ids for g in attention_groups] == [[0]]
|
||||
|
||||
@@ -1319,12 +1319,10 @@ def test_inflight_finish_deferred_cleanup() -> None:
|
||||
# Test 8: Null GPU blocks are skipped in store and load transfer pairs
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_multi_group_null_blocks_skipped() -> None:
|
||||
"""Null GPU blocks (no block_hash) must not appear in store or load pairs.
|
||||
"""The null block id never appears in store or load pairs.
|
||||
|
||||
In eager store mode, _prepare_eager_store_specs skips blocks whose
|
||||
block_hash is None (null blocks have no hash). We verify this by mixing
|
||||
real hashed blocks with unhashed (null-like) blocks in a single group and
|
||||
checking that only real blocks appear in the store list.
|
||||
A null block-table slot whose hash is still cached on the GPU is stored
|
||||
from the cached block instead; the null block itself is never copied.
|
||||
"""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, num_groups=1, lazy=False)
|
||||
sched = fix.scheduler
|
||||
@@ -1356,9 +1354,11 @@ def test_multi_group_null_blocks_skipped() -> None:
|
||||
f"Null block id {null_block_id} should not appear in store transfer pairs"
|
||||
)
|
||||
|
||||
# Only real block should be scheduled for store
|
||||
assert len(meta.store_gpu_blocks) == 1
|
||||
# The nulled slot's hash is still cached on the GPU, so that block is
|
||||
# recovered from the pool and stored alongside the real one.
|
||||
assert len(meta.store_gpu_blocks) == 2
|
||||
assert gpu_blocks[0].block_id in meta.store_gpu_blocks
|
||||
assert gpu_blocks[1].block_id in meta.store_gpu_blocks
|
||||
|
||||
# Complete the store
|
||||
assert meta.store_event >= 0
|
||||
@@ -1374,13 +1374,12 @@ def test_multi_group_null_blocks_skipped() -> None:
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
|
||||
# Only 1 block was stored (the real one)
|
||||
assert hit_tokens == BLOCK_SIZE
|
||||
assert hit_tokens == 2 * BLOCK_SIZE
|
||||
assert is_async is True
|
||||
|
||||
# Allocate new GPU blocks for the load
|
||||
gpu_blocks2 = gpu_pool.get_new_blocks(1)
|
||||
kv_blocks2 = KVCacheBlocks(blocks=([gpu_blocks2[0], null_block],))
|
||||
gpu_blocks2 = gpu_pool.get_new_blocks(2)
|
||||
kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,))
|
||||
sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens)
|
||||
|
||||
sched_out2 = make_scheduler_output({req2.request_id: 1})
|
||||
@@ -2800,3 +2799,47 @@ def test_boundary_handoff_keeps_block_meta_index_parallel() -> None:
|
||||
assert req_ids == [req.request_id]
|
||||
assert block_meta is not None
|
||||
assert len(block_meta) == len(gpu_ids) == len(cpu_ids)
|
||||
|
||||
|
||||
def test_finished_eager_store_recovers_nulled_window_tail() -> None:
|
||||
"""A sliding-window group nulls pages that left the window before the
|
||||
connector sees the finished request's block table, while the GPU keeps
|
||||
them hashed in its free queue. The eager store must copy those pages, or
|
||||
the CPU lookup (which needs the whole window) misses every time."""
|
||||
fix = make_scheduler(num_cpu_blocks=16, num_gpu_blocks=32, num_groups=2, lazy=False)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
null_block = gpu_pool.null_block
|
||||
|
||||
num_blocks = 4 # == sliding window of the test SWA group
|
||||
req = make_request(num_blocks=num_blocks)
|
||||
fa_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0)
|
||||
swa_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=1)
|
||||
# remove_skipped_blocks() ran: the two oldest window pages are freed (still
|
||||
# hashed) and their slots hold the null block.
|
||||
gpu_pool.free_blocks(swa_blocks[:2])
|
||||
swa_table = [null_block, null_block, *swa_blocks[2:]]
|
||||
kv_blocks = KVCacheBlocks(blocks=(fa_blocks, swa_table))
|
||||
req.num_computed_tokens = num_blocks * BLOCK_SIZE
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
sched.request_finished_all_groups(req, block_ids)
|
||||
meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
|
||||
assert null_block.block_id not in meta.store_gpu_blocks
|
||||
assert set(meta.store_gpu_blocks) == {b.block_id for b in (*fa_blocks, *swa_blocks)}
|
||||
assert meta.store_event >= 0
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
req2 = Request(
|
||||
request_id="req-window-tail-load",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
|
||||
assert hit_tokens == num_blocks * BLOCK_SIZE
|
||||
assert is_async is True
|
||||
|
||||
Reference in New Issue
Block a user