[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -364,6 +364,12 @@ def main() -> None:
|
||||
type=int,
|
||||
help="Maximum number of concurrent requests",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--request-timeout-seconds",
|
||||
type=float,
|
||||
default=600,
|
||||
help="Timeout for each request, including time waiting for a connection",
|
||||
)
|
||||
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -377,6 +383,7 @@ def main() -> None:
|
||||
temperature=args.temperature,
|
||||
seed=args.seed,
|
||||
max_concurrency=args.max_concurrency,
|
||||
request_timeout_seconds=args.request_timeout_seconds,
|
||||
)
|
||||
|
||||
# Print results to terminal
|
||||
|
||||
@@ -217,7 +217,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||
assert (out_ref - out_view).abs().max().item() == 0.0
|
||||
|
||||
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
@pytest.mark.parametrize("next_n", [1, 2, 4])
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
|
||||
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
|
||||
cache, with finite nonzero output."""
|
||||
@@ -240,7 +241,10 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
n_layers = 3
|
||||
layer = 1
|
||||
|
||||
q = torch.randn(batch, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
|
||||
q = (
|
||||
torch.randn(batch, next_n, h_q, head_dim, device=dev, dtype=torch.bfloat16)
|
||||
* 0.1
|
||||
)
|
||||
|
||||
# Structurally valid fp8 ds_mla payload: 512B fp8 + 16B f32 scales + 128B
|
||||
# bf16 rope (random bytes corrupt the scale region and yield NaNs).
|
||||
@@ -273,13 +277,13 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
|
||||
# Sparse indices: each batch uses its own disjoint blocks.
|
||||
blocks_per_batch = num_blocks // batch
|
||||
idx = torch.full((batch, 1, topk), -1, device=dev, dtype=torch.int32)
|
||||
idx = torch.full((batch, next_n, topk), -1, device=dev, dtype=torch.int32)
|
||||
for b in range(batch):
|
||||
slots: list[int] = []
|
||||
for blk in range(b * blocks_per_batch, (b + 1) * blocks_per_batch):
|
||||
slots.extend(blk * page + off for off in range(page))
|
||||
slots_t = torch.tensor(slots[:topk], device=dev, dtype=torch.int32)
|
||||
idx[b, 0, : slots_t.numel()] = slots_t
|
||||
idx[b, :, : slots_t.numel()] = slots_t
|
||||
|
||||
def run(kc):
|
||||
meta, num_splits = fm.get_mla_metadata()
|
||||
|
||||
@@ -316,6 +316,48 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
def test_cp_gather_fp8_mixes_resident_and_pinned_host_rows():
|
||||
seq_lens = [7, 5]
|
||||
block_size = 4
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
workspace_starts,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
expected,
|
||||
) = _build_test_case(seq_lens, block_size)
|
||||
|
||||
device_cache = cache.clone()
|
||||
host_cache = cache.cpu().pin_memory().view(-1, ENTRY_BYTES)
|
||||
device_rows = torch.arange(host_cache.shape[0], dtype=torch.int32, device="cuda")
|
||||
use_device = device_rows % 2 == 0
|
||||
host_rows = device_rows.clone()
|
||||
device_rows.masked_fill_(~use_device, -1)
|
||||
|
||||
host_cache[use_device.cpu()] = 0
|
||||
device_cache.view(-1, ENTRY_BYTES)[~use_device] = 0
|
||||
dst = torch.empty(
|
||||
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
device_cache,
|
||||
dst,
|
||||
block_table,
|
||||
workspace_starts,
|
||||
num_reqs,
|
||||
host_cache=host_cache,
|
||||
host_row_ids=host_rows,
|
||||
device_row_ids=device_rows,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
|
||||
)
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gather_seq_lens,seq_starts",
|
||||
[
|
||||
|
||||
@@ -234,6 +234,76 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str
|
||||
assert (topk == -1).all(), "topk buffer not cleared on indexer layer"
|
||||
|
||||
|
||||
def test_fused_norm_rope_packed_indexer_block_stride():
|
||||
"""Indexer writes use their HMA block stride and independent slot map."""
|
||||
torch.manual_seed(7)
|
||||
dev = "cuda"
|
||||
num_tokens = block_size = 2
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
|
||||
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
|
||||
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
|
||||
ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
cos_sin = make_cos_sin(32, ROPE_DIM, dev)
|
||||
|
||||
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4
|
||||
packed_block_stride = block_size * idx_row + 64
|
||||
backing = torch.zeros(2 * packed_block_stride, device=dev, dtype=torch.uint8)
|
||||
idx_cache = torch.as_strided(
|
||||
backing,
|
||||
(2, block_size, idx_row),
|
||||
(packed_block_stride, idx_row, 1),
|
||||
)
|
||||
mla_cache = torch.zeros(
|
||||
1,
|
||||
block_size,
|
||||
KV_LORA + ROPE_DIM,
|
||||
device=dev,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
topk = torch.zeros(num_tokens, 8, device=dev, dtype=torch.int32)
|
||||
|
||||
K.fused_norm_rope(
|
||||
pos,
|
||||
q_c,
|
||||
qw,
|
||||
EPS,
|
||||
kv_c,
|
||||
kvw,
|
||||
EPS,
|
||||
k_pe,
|
||||
cos_sin,
|
||||
ik,
|
||||
ikw,
|
||||
ikb,
|
||||
EPS,
|
||||
cos_sin,
|
||||
topk,
|
||||
slot_mapping=torch.arange(num_tokens, device=dev, dtype=torch.int64),
|
||||
indexer_slot_mapping=torch.arange(
|
||||
block_size, block_size + num_tokens, device=dev, dtype=torch.int64
|
||||
),
|
||||
indexer_k_cache=idx_cache,
|
||||
mla_kv_cache=mla_cache,
|
||||
has_indexer=True,
|
||||
)
|
||||
|
||||
ik_ref = rope(layer_norm(ik, ikw, ikb), pos, cos_sin, interleave=False)
|
||||
q_ref, s_ref = ue8m0_quant(ik_ref)
|
||||
packed = idx_cache[1].reshape(-1)
|
||||
values = (
|
||||
packed[: block_size * INDEX_HEAD_DIM].view(FP8).view(block_size, INDEX_HEAD_DIM)
|
||||
)
|
||||
scales = packed[block_size * INDEX_HEAD_DIM :].view(torch.float32)
|
||||
assert_fp8(values, q_ref, "packed indexer-K fp8")
|
||||
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
|
||||
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
|
||||
|
||||
|
||||
def test_fused_norm_rope_materializes_qk_without_local_cache_slots():
|
||||
"""DCP non-owner ranks still need valid query shards for query AllGather."""
|
||||
torch.manual_seed(7)
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
import vllm.models.deepseek_v32.attention as deepseek_attention
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
@@ -16,6 +17,51 @@ INDEXER_LAYER = "model.layers.0.self_attn.indexer.k_cache"
|
||||
MLA_LAYER = "model.layers.0.self_attn.attn"
|
||||
|
||||
|
||||
def test_sparse_attention_refreshes_batch_state_inside_eager_segment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
metadata = SimpleNamespace(num_actual_tokens=1)
|
||||
|
||||
class BatchStateRefreshed(Exception):
|
||||
pass
|
||||
|
||||
def refresh_batch_state(attn_metadata):
|
||||
assert attn_metadata is metadata
|
||||
raise BatchStateRefreshed
|
||||
|
||||
layer = SimpleNamespace(
|
||||
indexer=None,
|
||||
skip_topk=True,
|
||||
layer_name=MLA_LAYER,
|
||||
impl=SimpleNamespace(
|
||||
prepare_for_batch=refresh_batch_state,
|
||||
record_logical_topk_ready=lambda: None,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deepseek_attention,
|
||||
"get_attention_context",
|
||||
lambda layer_name: (metadata, None, torch.empty(1), None),
|
||||
)
|
||||
|
||||
with pytest.raises(BatchStateRefreshed):
|
||||
DeepseekV32Attention._sparse_indexer_and_attn(
|
||||
layer,
|
||||
torch.empty(1, dtype=torch.long),
|
||||
torch.empty(1, 1),
|
||||
torch.empty(1, 1, 1),
|
||||
torch.empty(1, 1, 1),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
torch.empty(1, 1, 1),
|
||||
torch.empty(1, 1, 1),
|
||||
torch.empty(1, 1),
|
||||
)
|
||||
|
||||
|
||||
def make_indexer_metadata(
|
||||
*,
|
||||
num_decodes: int = 0,
|
||||
@@ -246,6 +292,10 @@ def test_deepseek_v32_dispatches_selected_mha(
|
||||
_fp8_query=fp8_query,
|
||||
_use_sparse_mha=lambda _: True,
|
||||
rotary_emb=lambda _positions, q: (q + 1, None),
|
||||
impl=SimpleNamespace(
|
||||
record_logical_topk_ready=lambda: None,
|
||||
prepare_for_batch=lambda _: None,
|
||||
),
|
||||
forward_impl=record_forward_impl,
|
||||
)
|
||||
q_nope = torch.randn(2, 1, 2)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from inspect import signature
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
@@ -76,8 +77,16 @@ def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
|
||||
assert token_counts == (32,)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
|
||||
@pytest.mark.parametrize("skip_attn", [False, True])
|
||||
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2Runner
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as V1Runner
|
||||
|
||||
runner = _make_runner([])
|
||||
dummy_run_signature = signature((V2Runner if skip_attn else V1Runner)._dummy_run)
|
||||
runner._dummy_run.side_effect = lambda **kwargs: dummy_run_signature.bind(
|
||||
None, **kwargs
|
||||
)
|
||||
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
|
||||
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
|
||||
|
||||
@@ -93,7 +102,7 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
|
||||
) as get_buckets,
|
||||
patch("vllm.utils.flashinfer.autotune") as autotune,
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
|
||||
|
||||
assert get_buckets.call_args_list == [call(8192), call(128)]
|
||||
assert autotune.call_args_list == [
|
||||
@@ -106,11 +115,13 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
call(
|
||||
num_tokens=128,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
]
|
||||
|
||||
+66
-2
@@ -19,10 +19,12 @@ import vllm.config.vllm as vllm_config_module
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
DeviceConfig,
|
||||
EngramConfig,
|
||||
HiSparseConfig,
|
||||
KernelConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
@@ -279,6 +281,62 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
|
||||
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
|
||||
|
||||
|
||||
def test_hisparse_requires_v2_model_runner():
|
||||
config = object.__new__(VllmConfig)
|
||||
config.attention_config = AttentionConfig(hisparse_config=HiSparseConfig())
|
||||
|
||||
with patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", None):
|
||||
assert config.use_v2_model_runner
|
||||
with (
|
||||
patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", False),
|
||||
pytest.raises(ValueError, match="requires Model Runner V2"),
|
||||
):
|
||||
_ = config.use_v2_model_runner
|
||||
|
||||
|
||||
def test_hisparse_rejects_decode_context_parallelism(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
|
||||
with pytest.raises(ValueError, match="decode context parallelism"):
|
||||
VllmConfig(
|
||||
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=2,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_rejects_pipeline_parallelism(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
|
||||
with pytest.raises(ValueError, match="pipeline parallelism"):
|
||||
VllmConfig(
|
||||
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
|
||||
with pytest.raises(ValueError, match="requires the hybrid KV cache manager"):
|
||||
VllmConfig(
|
||||
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_rejects_non_cuda(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
|
||||
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
|
||||
VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig()))
|
||||
|
||||
|
||||
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
"""ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled
|
||||
MRV1 paths and off breakable cudagraphs by default."""
|
||||
@@ -305,7 +363,8 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
# (warning_once args must be hashable for its lru_cache).
|
||||
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
|
||||
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]),
|
||||
attention_config=AttentionConfig(),
|
||||
)
|
||||
assert VllmConfig.use_v2_model_runner.fget(config) is False
|
||||
finally:
|
||||
@@ -385,6 +444,7 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
model_config=model_config,
|
||||
attention_config=AttentionConfig(),
|
||||
speculative_config=SimpleNamespace(method="mtp") if with_mtp else None,
|
||||
parallel_config=SimpleNamespace(prefill_context_parallel_size=1),
|
||||
compilation_config=CompilationConfig(
|
||||
@@ -897,7 +957,10 @@ def test_models_default_to_v2_model_runner(model_config, expected, monkeypatch):
|
||||
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
|
||||
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
|
||||
config = SimpleNamespace(model_config=model_config)
|
||||
config = SimpleNamespace(
|
||||
model_config=model_config,
|
||||
attention_config=AttentionConfig(),
|
||||
)
|
||||
config._get_v2_model_runner_unsupported_features = lambda: []
|
||||
|
||||
assert VllmConfig.use_v2_model_runner.fget(config) is expected
|
||||
@@ -2787,6 +2850,7 @@ def test_draft_sample_method_gumbel_is_rejected():
|
||||
def _watermarked_vllm_config() -> VllmConfig:
|
||||
config = object.__new__(VllmConfig)
|
||||
config.watermark_config = WatermarkConfig(key=42)
|
||||
config.attention_config = AttentionConfig()
|
||||
config.speculative_config = None
|
||||
return config
|
||||
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for per-KV-group attention backend selection (backend_per_kind)."""
|
||||
"""Tests for attention configuration and per-KV-group backend selection."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.attention import AttentionConfig
|
||||
from vllm.config.attention import AttentionConfig, HiSparseConfig
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
_largest_kernel_block_within,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionType, MultipleOf
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import get_attn_spec_kind
|
||||
from vllm.v1.hisparse.runtime import ResolvedHiSparseConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpecKind
|
||||
|
||||
|
||||
@@ -86,3 +89,52 @@ def test_backend_per_kind_rejects_unknown_kind():
|
||||
|
||||
def test_backend_per_kind_defaults_empty():
|
||||
assert AttentionConfig().backend_per_kind == {}
|
||||
|
||||
|
||||
def test_hisparse_device_buffer_size_boundaries():
|
||||
vllm_config = SimpleNamespace(
|
||||
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
|
||||
speculative_config=None,
|
||||
)
|
||||
resolved = ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
|
||||
assert resolved is not None
|
||||
assert resolved.device_buffer_size == 256
|
||||
|
||||
vllm_config.attention_config.hisparse_config = HiSparseConfig(
|
||||
device_buffer_size=127
|
||||
)
|
||||
with pytest.raises(ValueError, match="expected at least 128"):
|
||||
ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
|
||||
|
||||
vllm_config.attention_config.hisparse_config = HiSparseConfig(
|
||||
device_buffer_size=32768
|
||||
)
|
||||
resolved = ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
|
||||
assert resolved is not None
|
||||
assert resolved.device_buffer_size == 32768
|
||||
|
||||
vllm_config.attention_config.hisparse_config = HiSparseConfig(
|
||||
device_buffer_size=32769
|
||||
)
|
||||
with pytest.raises(ValueError, match="int16 slot-index limit"):
|
||||
ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
|
||||
|
||||
|
||||
def test_hisparse_device_buffer_covers_speculative_window():
|
||||
vllm_config = SimpleNamespace(
|
||||
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=3,
|
||||
parallel_drafting=False,
|
||||
),
|
||||
)
|
||||
|
||||
resolved = ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
|
||||
assert resolved is not None
|
||||
assert resolved.device_buffer_size == 5 * 128
|
||||
|
||||
vllm_config.attention_config.hisparse_config = HiSparseConfig(
|
||||
device_buffer_size=4 * 128 - 1
|
||||
)
|
||||
with pytest.raises(ValueError, match="expected at least 512"):
|
||||
ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
|
||||
|
||||
@@ -200,6 +200,7 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype(
|
||||
layer = SimpleNamespace(
|
||||
kv_cache_dtype=cache_dtype,
|
||||
head_size=576,
|
||||
indexer=None,
|
||||
non_causal_multi_token_decode=False,
|
||||
sliding_window=None,
|
||||
)
|
||||
|
||||
@@ -55,6 +55,7 @@ def build_chunked_context(
|
||||
def test_sparse_context_lengths_do_not_force_gpu_sync(monkeypatch):
|
||||
"""Sparse MLA must pin computed context lengths before the H2D copy."""
|
||||
builder = SimpleNamespace(
|
||||
_context_lens_cpu=torch.empty(2, dtype=torch.int32, pin_memory=True),
|
||||
chunked_prefill_workspace=torch.empty((2048, 1)),
|
||||
chunked_prefill_workspace_size=1024,
|
||||
kv_cache_spec=SimpleNamespace(block_size=BLOCK_SIZE),
|
||||
@@ -113,6 +114,7 @@ def test_chunks_gather_every_context_row_exactly_once(context_lens, workspace_si
|
||||
query_lens = [4] * len(context_lens)
|
||||
metadata = build_chunked_context(context_lens, query_lens, workspace_size)
|
||||
assert metadata is not None
|
||||
assert metadata.context_lens.tolist() == context_lens
|
||||
|
||||
gathered: dict[int, list[tuple[int, int]]] = {}
|
||||
previous_request_start = -1
|
||||
@@ -253,6 +255,7 @@ def test_dcp_chunks_fit_the_per_rank_row_budget():
|
||||
dcp_local_block_size=interleave,
|
||||
)
|
||||
assert metadata is not None
|
||||
assert metadata.context_lens.tolist() == context_lens
|
||||
virtual_block_size = interleave * dcp_world_size
|
||||
|
||||
local_cursor: dict[int, int] = {}
|
||||
@@ -330,3 +333,23 @@ def test_dcp_reorg_uses_each_chunks_local_starts():
|
||||
)
|
||||
|
||||
torch.testing.assert_close(reorganized, torch.cat(expected))
|
||||
|
||||
|
||||
def test_chunked_context_metadata_owns_context_lengths():
|
||||
"""Reusing CPU scratch must not overwrite an earlier batch's context lengths."""
|
||||
context_lens = torch.tensor([16, 32], dtype=torch.int32)
|
||||
metadata = build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens,
|
||||
prefill_query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32),
|
||||
chunked_prefill_workspace=torch.empty((64, 1)),
|
||||
chunked_prefill_workspace_size=64,
|
||||
block_size=BLOCK_SIZE,
|
||||
align_chunk_to_block=True,
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=1,
|
||||
dcp_local_block_size=1,
|
||||
dcp_virtual_block_size=1,
|
||||
)
|
||||
context_lens.zero_()
|
||||
assert metadata is not None
|
||||
assert metadata.context_lens.tolist() == [16, 32]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -41,7 +41,9 @@ def test_connector_without_divergent_hit_support_uses_common_lookup():
|
||||
manager = MagicMock()
|
||||
manager.get_computed_blocks.return_value = (common_blocks, 0, 0)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=False),
|
||||
connector=SimpleNamespace(
|
||||
supports_divergent_local_hybrid_hits=False,
|
||||
),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
@@ -61,7 +63,9 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
|
||||
True,
|
||||
)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=True),
|
||||
connector=SimpleNamespace(
|
||||
supports_divergent_local_hybrid_hits=True,
|
||||
),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
@@ -922,71 +926,6 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
|
||||
scheduler.kv_cache_manager.remove_skipped_blocks.assert_called_once()
|
||||
|
||||
|
||||
def test_block_pool_touch_pins_released_cow_target():
|
||||
"""The connector can rescue an offered CoW target after its step-scoped
|
||||
retention is released by using the bound BlockPool's touch method."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
# Retention released before the drain (defer_block_free=False ordering).
|
||||
_copies, retained = manager.take_kv_cache_block_copies()
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((_group_id, block_id, boundary_tokens),) = offloads["0"]
|
||||
assert boundary_tokens == 6
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
assert cow_block.ref_cnt == 0
|
||||
assert block_id in _free_block_ids(manager)
|
||||
|
||||
manager.block_pool.touch([cow_block])
|
||||
assert cow_block.ref_cnt == 1
|
||||
assert block_id not in _free_block_ids(manager)
|
||||
|
||||
# The connector-pinned block is out of the free queue: draining every free block
|
||||
# neither trips the allocator's ref_cnt assert nor hands it out.
|
||||
new_blocks = manager.block_pool.get_new_blocks(
|
||||
manager.block_pool.get_num_free_blocks()
|
||||
)
|
||||
assert block_id not in {b.block_id for b in new_blocks}
|
||||
|
||||
|
||||
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar
|
||||
from vllm.v1.utils import ConstantList
|
||||
|
||||
from .utils import create_requests, create_scheduler
|
||||
from .utils import create_requests, create_scheduler, mock_kv
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -145,6 +145,32 @@ def test_abort():
|
||||
assert req.num_output_tokens == abort_order_copy.index(i)
|
||||
|
||||
|
||||
def test_connector_metadata_precedes_async_placeholder_advance(monkeypatch):
|
||||
"""Mirroring must see earlier unresolved outputs, excluding the current step."""
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True,
|
||||
use_kv_connector=mock_kv(matched_tokens=0, is_async=False),
|
||||
)
|
||||
(request,) = create_requests(num_requests=1, num_tokens=4, max_tokens=4)
|
||||
scheduler.add_request(request)
|
||||
|
||||
assert scheduler.connector is not None
|
||||
build_connector_meta = scheduler.connector.build_connector_meta
|
||||
observed_placeholders = []
|
||||
|
||||
def record_placeholders(scheduler_output):
|
||||
observed_placeholders.append(request.num_output_placeholders)
|
||||
return build_connector_meta(scheduler_output)
|
||||
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "build_connector_meta", record_placeholders
|
||||
)
|
||||
scheduler.schedule()
|
||||
scheduler.schedule()
|
||||
|
||||
assert observed_placeholders == [0, 1]
|
||||
|
||||
|
||||
def test_preempt():
|
||||
scheduler = create_scheduler(async_scheduling=True)
|
||||
requests = create_requests(num_requests=10, max_tokens=20)
|
||||
|
||||
@@ -75,6 +75,7 @@ def _mock_vllm_config(layout: str | None):
|
||||
config.cache_config = CacheConfig()
|
||||
config.cache_config.num_gpu_blocks_override = None
|
||||
config.cache_config.kv_cache_layout = layout
|
||||
config.attention_config.hisparse_config = None
|
||||
return config
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.config import (
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.attention import HiSparseConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -52,10 +53,16 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
make_block_hash_with_group_id,
|
||||
tensor_data,
|
||||
)
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
get_hisparse_gpu_memory_usage,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -68,18 +75,139 @@ from vllm.v1.kv_cache_interface import (
|
||||
SinkFullAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
SparseCacheRole,
|
||||
UniformTypeKVCacheSpecs,
|
||||
get_kv_cache_spec_kind,
|
||||
get_kv_cache_spec_sliding_window,
|
||||
is_full_attention_spec,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.metrics.stats import CachingMetrics, PrefixCacheStats
|
||||
from vllm.v1.request import Request
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_block_size", [32, 64])
|
||||
def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
specs = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=gpu_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
|
||||
block_size=gpu_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert group_spec is not None
|
||||
group = KVCacheGroupSpec(list(specs), group_spec)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(index_topk=128),
|
||||
max_model_len=gpu_block_size,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
cache_config=SimpleNamespace(
|
||||
num_gpu_blocks_override=7,
|
||||
prefix_cache_retention_interval=None,
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
|
||||
),
|
||||
)
|
||||
indexer_spec = specs["model.layers.0.self_attn.indexer"]
|
||||
assert get_hisparse_gpu_memory_usage(config, [group]) == (
|
||||
indexer_spec.max_memory_usage_bytes(config)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, [group], available_memory=2**30
|
||||
)
|
||||
assert cache_config.num_blocks == 7
|
||||
assert cache_config.hisparse_host_num_blocks is not None
|
||||
assert cache_config.hisparse_host_num_blocks > 7
|
||||
|
||||
host_group, indexer_group, *auxiliary_groups = cache_config.kv_cache_groups
|
||||
assert host_group.host_resident
|
||||
assert not any(group.host_resident for group in [indexer_group, *auxiliary_groups])
|
||||
host_layers = set(host_group.layer_names)
|
||||
for tensor in cache_config.kv_cache_tensors:
|
||||
assert all(
|
||||
(name in host_layers) == tensor.host_resident for name in tensor.layers
|
||||
)
|
||||
host_tensor = next(t for t in cache_config.kv_cache_tensors if t.host_resident)
|
||||
host_spec = host_group.kv_cache_spec.kv_cache_specs[host_tensor.layers[0]]
|
||||
assert host_tensor.block_stride == host_spec.page_size_bytes
|
||||
assert host_tensor.layer_stride == (
|
||||
host_spec.page_size_bytes * cache_config.hisparse_host_num_blocks
|
||||
)
|
||||
assert host_group.kv_cache_spec.block_size == gpu_block_size
|
||||
assert indexer_group.kv_cache_spec.block_size == gpu_block_size
|
||||
host_specs = host_group.kv_cache_spec.kv_cache_specs
|
||||
gpu_indexer_specs = indexer_group.kv_cache_spec.kv_cache_specs
|
||||
assert set(host_specs) == {"model.layers.0.self_attn"}
|
||||
assert set(gpu_indexer_specs) == {"model.layers.0.self_attn.indexer"}
|
||||
assert indexer_group.kv_cache_spec.prefix_cacheable
|
||||
assert host_group.enable_kv_transfer
|
||||
auxiliary_specs = [group.kv_cache_spec for group in auxiliary_groups]
|
||||
assert any(isinstance(spec, HiSparseResidentSpec) for spec in auxiliary_specs)
|
||||
assert any(isinstance(spec, HiSparseHotSpec) for spec in auxiliary_specs)
|
||||
assert all(
|
||||
spec.block_size == gpu_block_size
|
||||
for spec in auxiliary_specs
|
||||
if isinstance(spec, (HiSparseResidentSpec, HiSparseHotSpec))
|
||||
)
|
||||
scheduler_block_size, hash_block_size = kv_cache_utils.resolve_kv_cache_block_sizes(
|
||||
cache_config,
|
||||
SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
kv_transfer_config=object(),
|
||||
),
|
||||
)
|
||||
assert scheduler_block_size == hash_block_size == gpu_block_size
|
||||
|
||||
|
||||
def test_hisparse_rejects_deepseek_v4():
|
||||
full_specs = {
|
||||
"model.layers.0.attn": MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.uint8,
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
}
|
||||
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
|
||||
assert full_uniform is not None
|
||||
group = KVCacheGroupSpec(list(full_specs), full_uniform)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
|
||||
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
|
||||
create_hisparse_layout(
|
||||
config,
|
||||
[group],
|
||||
host_budget=2**30,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _auto_init_hash_fn(request):
|
||||
hash_fn: Callable
|
||||
@@ -3848,6 +3976,22 @@ def test_iter_layer_specs_returns_group_members():
|
||||
assert list(iter_layer_specs(wrapped)) == [full, mla]
|
||||
|
||||
|
||||
def test_wrapped_mamba_group_requires_block_zeroing():
|
||||
mamba = MambaSpec(
|
||||
block_size=4,
|
||||
shapes=((4, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
)
|
||||
wrapped = UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs={"mamba": mamba})
|
||||
config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["mamba"], wrapped)],
|
||||
)
|
||||
|
||||
assert config.needs_kv_cache_zeroing
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark", model_type=None):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,12 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
HiSparseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalKwargsItem,
|
||||
@@ -34,6 +39,7 @@ from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
@@ -148,6 +154,30 @@ def test_get_num_unfinished_requests():
|
||||
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
|
||||
|
||||
|
||||
def _bind_hisparse_connector(scheduler):
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
|
||||
|
||||
coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16))
|
||||
connector.connector_scheduler.bind_coordinator(coordinator)
|
||||
composite = object.__new__(MultiConnector)
|
||||
composite._connectors = (connector,)
|
||||
scheduler.connector = composite
|
||||
return coordinator
|
||||
|
||||
|
||||
def test_pending_hisparse_spill_keeps_scheduler_alive():
|
||||
"""A final host spill must complete after the last request finishes."""
|
||||
scheduler = create_scheduler()
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
pending = coordinator.pending_spills
|
||||
pending[0] = Mock()
|
||||
assert scheduler.has_requests()
|
||||
pending.clear()
|
||||
assert not scheduler.has_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enable_prefix_caching, prompt_logprobs",
|
||||
[
|
||||
@@ -1181,6 +1211,45 @@ def test_preempt_during_execution():
|
||||
assert requests[1].output_token_ids[0] == 42
|
||||
|
||||
|
||||
def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
|
||||
"""Pending reclamation must stall rather than preempt its owning request."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[0]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None)
|
||||
)
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "build_connector_meta", Mock(return_value=None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
coordinator,
|
||||
"has_pending_reclamation",
|
||||
Mock(return_value=True),
|
||||
)
|
||||
|
||||
deferred_output = scheduler.schedule()
|
||||
|
||||
assert not deferred_output.num_scheduled_tokens
|
||||
assert scheduler.running == [request]
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
assert request.num_preemptions == 0
|
||||
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import prometheus_client
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import fork_new_process_for_each_test
|
||||
from vllm.config import AttentionConfig, HiSparseConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
|
||||
MultiConnector,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V3.2"
|
||||
|
||||
HF_OVERRIDES = {
|
||||
"num_hidden_layers": 8,
|
||||
"hidden_size": 256,
|
||||
"intermediate_size": 512,
|
||||
"num_attention_heads": 8,
|
||||
"num_key_value_heads": 1,
|
||||
"n_routed_experts": 8,
|
||||
"num_experts_per_tok": 2,
|
||||
"index_topk": 128,
|
||||
}
|
||||
|
||||
|
||||
def _shrink_config(config):
|
||||
overrides = HF_OVERRIDES.copy()
|
||||
if any("MTP" in architecture for architecture in config.architectures):
|
||||
overrides.pop("num_hidden_layers")
|
||||
config.update(overrides)
|
||||
return config
|
||||
|
||||
|
||||
def _num_hisparse_spills(runner: VllmRunner) -> int:
|
||||
client = runner.llm.llm_engine.engine_core
|
||||
coordinator = get_hisparse_coordinator(
|
||||
client.engine_core.scheduler.kv_cache_manager
|
||||
)
|
||||
return coordinator.next_spill_id
|
||||
|
||||
|
||||
def _offload_load_bytes() -> float:
|
||||
return sum(
|
||||
sample.value
|
||||
for metric in prometheus_client.REGISTRY.collect()
|
||||
if metric.name == "vllm:kv_offload_load_bytes"
|
||||
for sample in metric.samples
|
||||
if sample.name == "vllm:kv_offload_load_bytes_total"
|
||||
)
|
||||
|
||||
|
||||
def _get_hisparse_worker(runner: VllmRunner) -> HiSparseConnectorWorker:
|
||||
engine_core = runner.llm.llm_engine.engine_core.engine_core
|
||||
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
|
||||
connector = model_runner.kv_connector.kv_connector
|
||||
if isinstance(connector, MultiConnector):
|
||||
connector = next(
|
||||
child
|
||||
for child in connector.sub_connectors
|
||||
if isinstance(child, HiSparseConnector)
|
||||
)
|
||||
assert isinstance(connector, HiSparseConnector)
|
||||
assert connector.connector_worker is not None
|
||||
return connector.connector_worker
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"with_offloading", [False, True], ids=["standalone", "offload"]
|
||||
)
|
||||
@fork_new_process_for_each_test
|
||||
def test_hisparse_spill_and_prefix_restore(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
vllm_runner: type[VllmRunner],
|
||||
with_offloading: bool,
|
||||
):
|
||||
"""Spilled prefixes restore and FULL-graph decode writes reach host KV.
|
||||
|
||||
A regression omitted attention metadata before FULL graph replay, so decode
|
||||
completed normally while its newly written KV rows were never copied to host.
|
||||
"""
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.major < 9:
|
||||
pytest.skip("Sparse MLA requires Hopper or newer")
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
|
||||
|
||||
target = [1000 + i % 64 for i in range(257)]
|
||||
pressure = [
|
||||
[2000 + request_idx * 128 + i % 64 for i in range(257)]
|
||||
for request_idx in range(4)
|
||||
]
|
||||
hisparse_connector = {
|
||||
"kv_connector": "HiSparseConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"host_pool_gib": 1},
|
||||
}
|
||||
kv_transfer_config = KVTransferConfig(**hisparse_connector)
|
||||
if with_offloading:
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={
|
||||
"connectors": [
|
||||
hisparse_connector,
|
||||
{
|
||||
"kv_connector": "OffloadingConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
|
||||
},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
load_format="dummy",
|
||||
hf_overrides=_shrink_config,
|
||||
attention_config=AttentionConfig(
|
||||
hisparse_config=HiSparseConfig(
|
||||
device_buffer_size=512,
|
||||
)
|
||||
),
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
block_size=64,
|
||||
max_model_len=320,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_seqs=4,
|
||||
num_gpu_blocks_override=128,
|
||||
disable_log_stats=False,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 3},
|
||||
compilation_config={
|
||||
"cudagraph_mode": "FULL_AND_PIECEWISE",
|
||||
"cudagraph_capture_sizes": [1, 4],
|
||||
},
|
||||
) as runner:
|
||||
worker = _get_hisparse_worker(runner)
|
||||
engine_core = runner.llm.llm_engine.engine_core.engine_core
|
||||
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
|
||||
full_graph_calls = 0
|
||||
full_replay_pending = False
|
||||
original_run_fullgraph = model_runner.cudagraph_manager.run_fullgraph
|
||||
|
||||
def record_fullgraph(batch_desc):
|
||||
nonlocal full_graph_calls, full_replay_pending
|
||||
full_graph_calls += 1
|
||||
full_replay_pending = True
|
||||
return original_run_fullgraph(batch_desc)
|
||||
|
||||
monkeypatch.setattr(
|
||||
model_runner.cudagraph_manager, "run_fullgraph", record_fullgraph
|
||||
)
|
||||
|
||||
original_finish_forward = worker.finish_forward
|
||||
verified_full_replay_rows = 0
|
||||
copied_nonzero_kv = False
|
||||
|
||||
def finish_forward():
|
||||
nonlocal full_replay_pending, verified_full_replay_rows
|
||||
nonlocal copied_nonzero_kv
|
||||
try:
|
||||
original_finish_forward()
|
||||
if not full_replay_pending:
|
||||
return
|
||||
# Synchronize only the test's inspection, not finish_forward itself.
|
||||
with gpu_sync_allowed():
|
||||
torch.accelerator.synchronize()
|
||||
assert worker._row_mirrors
|
||||
for layer_index, cache in enumerate(worker.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
for mirror in worker._row_mirrors:
|
||||
source_row = mirror.source_starts[source_index]
|
||||
destination_row = mirror.destination_start
|
||||
num_rows = mirror.num_rows
|
||||
source_block, source_offset = divmod(
|
||||
source_row, worker.kernel_block_size
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
gpu_rows = worker.resident_caches[layer_index][
|
||||
source_block, source_offset : source_offset + num_rows
|
||||
].cpu()
|
||||
host_rows = worker.host_caches[layer_index][
|
||||
destination_row : destination_row + num_rows
|
||||
]
|
||||
assert torch.equal(host_rows, gpu_rows)
|
||||
copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item())
|
||||
verified_full_replay_rows += num_rows
|
||||
finally:
|
||||
full_replay_pending = False
|
||||
|
||||
monkeypatch.setattr(worker, "finish_forward", finish_forward)
|
||||
|
||||
expected = runner.generate_greedy([target], max_tokens=8)
|
||||
|
||||
assert full_graph_calls > 0
|
||||
assert verified_full_replay_rows > 0
|
||||
assert copied_nonzero_kv
|
||||
|
||||
runner.generate_greedy(pressure, max_tokens=8)
|
||||
|
||||
assert _num_hisparse_spills(runner) > 0
|
||||
load_bytes = _offload_load_bytes()
|
||||
actual = runner.generate_greedy([target], max_tokens=8)
|
||||
runner.generate_greedy([[42]], max_tokens=1)
|
||||
|
||||
assert actual == expected
|
||||
if with_offloading:
|
||||
assert _offload_load_bytes() > load_bytes
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupRole,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVCacheTensor,
|
||||
@@ -336,6 +337,36 @@ def test_worker_kv_bytes_preserves_tensor_layout(packed: bool):
|
||||
assert offloading_config.cache.blocks_per_chunk == 2
|
||||
|
||||
|
||||
def test_hisparse_offloads_only_indexer_group():
|
||||
source = KVCacheGroupSpec(
|
||||
["source"],
|
||||
_full_attention_spec(),
|
||||
host_resident=True,
|
||||
)
|
||||
indexer = KVCacheGroupSpec(
|
||||
["indexer"],
|
||||
_full_attention_spec(),
|
||||
role=KVCacheGroupRole.HISPARSE_INDEXER,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[source, indexer],
|
||||
hisparse_host_num_blocks=4,
|
||||
)
|
||||
config = _make_vllm_config()
|
||||
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
scheduler_config = SchedulerOffloadConfig.from_spec(
|
||||
MockOffloadingSpec(offloading_config), config, kv_cache_config
|
||||
)
|
||||
|
||||
assert [
|
||||
(group.group_id, group.layer_names) for group in offloading_config.groups
|
||||
] == [(1, ("indexer",))]
|
||||
assert [group.group_idx for group in scheduler_config.kv_group_configs] == [1]
|
||||
|
||||
|
||||
def test_zero_blocks_skips_tensor_layout_validation():
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.num_blocks = 0
|
||||
@@ -792,7 +823,6 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
False,
|
||||
id="uniform-uncertifiable-inner",
|
||||
),
|
||||
pytest.param([], False, id="attention-free"),
|
||||
],
|
||||
)
|
||||
def test_canonical_layout_gate(kv_cache_groups, certified):
|
||||
|
||||
@@ -98,6 +98,14 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
)
|
||||
manager = CPUOffloadingManager(num_chunks=100)
|
||||
spec = SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
groups=tuple(
|
||||
SimpleNamespace(
|
||||
group_id=i, tokens_per_block=group.kv_cache_spec.block_size
|
||||
)
|
||||
for i, group in enumerate(groups)
|
||||
)
|
||||
),
|
||||
tokens_per_block=(256, 64, 8),
|
||||
tokens_per_hash=8,
|
||||
blocks_per_chunk=4,
|
||||
@@ -303,7 +311,24 @@ def test_aligned_boundary_store_uses_exact_source_with_partial_tail():
|
||||
assert src_spec.block_indices == [0, 0]
|
||||
|
||||
|
||||
def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
|
||||
def test_aligned_boundary_store_maps_sparse_group_id_to_dense_transfer_slot():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
indexer_config = scheduler.config.kv_group_configs[1]
|
||||
scheduler.config = scheduler.config._replace(kv_group_configs=(indexer_config,))
|
||||
_make_partial_tail_request(scheduler)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
jobs = scheduler._build_aligned_boundary_store_jobs({"req": [(1, 98, 16)]})
|
||||
|
||||
[job] = jobs.values()
|
||||
assert job.src_spec.group_sizes == [1]
|
||||
assert job.src_spec.block_indices == [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cow_reuse", [False, True])
|
||||
def test_aligned_boundary_store_flushes_before_block_reuse(cow_reuse):
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
@@ -320,7 +345,11 @@ def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
|
||||
[job_id] = meta.store_jobs
|
||||
|
||||
output = SchedulerOutput.make_empty()
|
||||
output.kv_cache_block_copies = [KVCacheBlockCopy(98, 99)]
|
||||
if cow_reuse:
|
||||
output.kv_cache_block_copies = [KVCacheBlockCopy(98, 99)]
|
||||
else:
|
||||
output.scheduled_cached_reqs.req_ids = ["req"]
|
||||
output.scheduled_cached_reqs.new_block_ids = [([], [99])]
|
||||
meta = scheduler.build_connector_meta(output)
|
||||
|
||||
assert meta.jobs_to_flush == {job_id}
|
||||
@@ -384,6 +413,20 @@ def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
|
||||
assert req_status.partial_tail_boundary is None
|
||||
|
||||
|
||||
def test_lookup_cap_stops_at_authoritative_prefix_boundary():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
request.skip_reading_prefix_cache = False
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
tokens, load_async = scheduler.get_num_new_matched_tokens(
|
||||
request, 0, max_num_new_tokens=20
|
||||
)
|
||||
|
||||
assert (tokens, load_async) == (20, True)
|
||||
assert scheduler._req_status["req"].partial_tail_boundary == 20
|
||||
|
||||
|
||||
def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
|
||||
"""A recurrent group may hold unhashed blocks inside the computed prefix.
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import (
|
||||
ExampleConnector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
HiSparseConnectorMetadata,
|
||||
HiSparseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
|
||||
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
def test_cache_manager_binding_preserves_hisparse_and_legacy_pool_hooks(nested):
|
||||
"""Composite binding must reach HiSparse and existing pool-only connectors."""
|
||||
hisparse = object.__new__(HiSparseConnector)
|
||||
hisparse._role = KVConnectorRole.SCHEDULER
|
||||
hisparse.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
legacy = object.__new__(ExampleConnector)
|
||||
legacy.bind_gpu_block_pool = MagicMock()
|
||||
connector = object.__new__(MultiConnector)
|
||||
connector._connectors = [hisparse, legacy]
|
||||
if nested:
|
||||
parent = object.__new__(MultiConnector)
|
||||
parent._connectors = [connector]
|
||||
connector = parent
|
||||
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
|
||||
|
||||
manager = make_hisparse_kv_cache_manager(16, 16)
|
||||
|
||||
connector.bind_kv_cache_manager(manager)
|
||||
|
||||
assert hisparse.connector_scheduler.coordinator is get_hisparse_coordinator(manager)
|
||||
legacy.bind_gpu_block_pool.assert_called_once_with(manager.block_pool)
|
||||
|
||||
|
||||
def test_hisparse_requires_block_outermost_device_layout():
|
||||
assert HiSparseConnector.get_required_kvcache_layout(MagicMock()) == "BLHNC"
|
||||
|
||||
|
||||
def test_no_forward_enqueues_deferred_hisparse_transfers():
|
||||
"""A zero-token step must still enqueue deferred post-forward transfers."""
|
||||
connector = object.__new__(ActiveKVConnector)
|
||||
connector._disabled = False
|
||||
connector.pre_forward = MagicMock()
|
||||
connector.finish_forward = MagicMock()
|
||||
connector.post_forward = MagicMock(return_value=None)
|
||||
|
||||
scheduler_output = SimpleNamespace(finished_req_ids=set())
|
||||
connector.no_forward(scheduler_output)
|
||||
|
||||
connector.pre_forward.assert_called_once_with(scheduler_output)
|
||||
connector.finish_forward.assert_called_once_with()
|
||||
|
||||
|
||||
def test_full_graph_step_prepares_host_mirror_outside_model():
|
||||
"""Graph replay must restore host-mirror state cleared at step start."""
|
||||
runtime = SimpleNamespace(
|
||||
is_group_leader=True,
|
||||
eager_host_mirror=True,
|
||||
begin_forward=MagicMock(),
|
||||
invalidate_written_slots=MagicMock(),
|
||||
)
|
||||
handle = HiSparseCacheHandle(runtime)
|
||||
handle.mirror_slot_mapping = torch.tensor([4, 5])
|
||||
worker = object.__new__(HiSparseConnectorWorker)
|
||||
worker.cache_layer_names = ["layer"]
|
||||
worker.cache_handles = [handle]
|
||||
worker._group_leaders = (("layer", handle),)
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker.is_host_writer = True
|
||||
worker._enqueue_row_dma = MagicMock()
|
||||
worker.start_step = MagicMock(
|
||||
side_effect=lambda *_args, **_kwargs: worker._clear_forward_mirror_state()
|
||||
)
|
||||
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_worker = worker
|
||||
connector._get_connector_metadata = MagicMock(
|
||||
return_value=HiSparseConnectorMetadata(None, (), (), {}, True)
|
||||
)
|
||||
req_id_per_token = torch.tensor([0, 1], dtype=torch.int32)
|
||||
attn_metadata = SimpleNamespace(
|
||||
num_actual_tokens=2,
|
||||
num_decode_tokens=2,
|
||||
num_reqs=2,
|
||||
max_query_len=1,
|
||||
req_id_per_token=req_id_per_token,
|
||||
)
|
||||
|
||||
connector.start_load_kv(
|
||||
SimpleNamespace(),
|
||||
request_state_indices=None,
|
||||
request_ids=[],
|
||||
attn_metadata={"layer": attn_metadata},
|
||||
)
|
||||
|
||||
worker._enqueue_host_mirror()
|
||||
|
||||
worker._enqueue_row_dma.assert_called_once_with((0,), ready_event=None)
|
||||
runtime.invalidate_written_slots.assert_called_once()
|
||||
@@ -3297,7 +3297,9 @@ def _make_bare_worker(
|
||||
# path; everything flows through the coordinator).
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
|
||||
worker.disk_offload_buffer_budget_bytes = None
|
||||
@@ -3311,6 +3313,18 @@ def _make_bare_worker(
|
||||
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
group = KVCacheGroupSpec(["layer0", "__cross_layer__"], spec)
|
||||
worker._kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_gpu_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=0,
|
||||
layers=["layer0", "__cross_layer__"],
|
||||
layer_stride=0,
|
||||
block_stride=0,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[group],
|
||||
)
|
||||
worker._kv_cache_groups = [group]
|
||||
worker.pcp_size = 1
|
||||
worker.dcp_size = 1
|
||||
@@ -4059,6 +4073,77 @@ def test_register_kv_caches_shared_storage(layout: KVCacheLayout):
|
||||
worker.store.register_buffer.assert_called_once_with(raw.data_ptr(), raw.nbytes)
|
||||
|
||||
|
||||
def test_register_kv_caches_uses_transfer_group_memory_domain():
|
||||
"""Derived GPU caches must not change host-source transfer addresses."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupRole,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
|
||||
host_num_blocks = 4
|
||||
gpu_num_blocks = 2
|
||||
page_size = 32
|
||||
source_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float16
|
||||
)
|
||||
indexer_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float16
|
||||
)
|
||||
source_group = KVCacheGroupSpec(
|
||||
["source"],
|
||||
source_spec,
|
||||
host_resident=True,
|
||||
)
|
||||
indexer_group = KVCacheGroupSpec(
|
||||
["indexer"],
|
||||
indexer_spec,
|
||||
enable_kv_transfer=False,
|
||||
role=KVCacheGroupRole.HISPARSE_INDEXER,
|
||||
)
|
||||
config = KVCacheConfig(
|
||||
num_blocks=gpu_num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=host_num_blocks * page_size,
|
||||
layers=["source"],
|
||||
layer_stride=host_num_blocks * page_size,
|
||||
block_stride=page_size,
|
||||
host_resident=True,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=gpu_num_blocks * page_size,
|
||||
layers=["indexer"],
|
||||
layer_stride=gpu_num_blocks * page_size,
|
||||
block_stride=page_size,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[source_group, indexer_group],
|
||||
hisparse_host_num_blocks=host_num_blocks,
|
||||
)
|
||||
worker = _make_bare_worker(num_gpu_blocks=gpu_num_blocks)
|
||||
worker._kv_cache_config = config
|
||||
worker._kv_cache_groups = list(config.transfer_groups)
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(KeyMetadata("test-model", 0, 0, 0, 0), block_size=16)
|
||||
]
|
||||
source = torch.zeros(host_num_blocks, page_size, dtype=torch.uint8)
|
||||
indexer = torch.zeros(gpu_num_blocks, page_size, dtype=torch.uint8)
|
||||
|
||||
_register_with_mocked_threads(
|
||||
worker,
|
||||
{"source": source, "indexer": indexer},
|
||||
)
|
||||
|
||||
db = worker.token_dbs[0]
|
||||
assert db.kv_caches_base_addr == [source.data_ptr()]
|
||||
assert db.block_len == [page_size]
|
||||
worker.store.register_buffer.assert_called_once_with(
|
||||
source.data_ptr(), source.nbytes
|
||||
)
|
||||
|
||||
|
||||
def test_register_kv_caches_separate_head_groups():
|
||||
# LHBNC gives each head group its own region; the K/V split doubles heads.
|
||||
layout = KVCacheLayout.LHBNC
|
||||
|
||||
@@ -304,10 +304,10 @@ def test_multi_example_connector_consistency():
|
||||
events = get_connector_events()
|
||||
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
|
||||
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
|
||||
# Initial events bind the block pool, query completion counts, and exchange
|
||||
# Initial events bind the cache manager, query completion counts, and exchange
|
||||
# handshake metadata before the request is enqueued.
|
||||
assert storage1_scheduler_events[:7] == [
|
||||
"bind_gpu_block_pool",
|
||||
"bind_kv_cache_manager",
|
||||
"get_finished_count",
|
||||
"set_xfer_handshake_metadata_pp_aware",
|
||||
"on_new_request",
|
||||
@@ -315,8 +315,8 @@ def test_multi_example_connector_consistency():
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
# First three events are from initialization. During generate(), layer hooks
|
||||
# run before the deferred load is started after the forward pass.
|
||||
# First three events are from initialization. Layer hooks run before the
|
||||
# deferred load starts after the forward pass.
|
||||
expected_worker_prefix = [
|
||||
"register_kv_caches",
|
||||
"set_host_xfer_buffer_ops",
|
||||
@@ -328,12 +328,12 @@ def test_multi_example_connector_consistency():
|
||||
]
|
||||
for connector_name in ("storage1-WORKER", "storage2-WORKER"):
|
||||
worker_events = events[connector_name]
|
||||
assert worker_events[:7] == expected_worker_prefix
|
||||
assert worker_events[: len(expected_worker_prefix)] == expected_worker_prefix
|
||||
assert worker_events.index("start_load_kv") > worker_events.index(
|
||||
"save_kv_layer"
|
||||
)
|
||||
assert storage2_scheduler_events[:7] == [
|
||||
"bind_gpu_block_pool",
|
||||
"bind_kv_cache_manager",
|
||||
"get_finished_count",
|
||||
"set_xfer_handshake_metadata_pp_aware",
|
||||
"on_new_request",
|
||||
|
||||
@@ -29,7 +29,10 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
get_current_attn_backend,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import nixl
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
|
||||
MultiKVConnectorStats,
|
||||
@@ -518,6 +521,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
slot_size_bytes = 4096
|
||||
self.slot_size_per_layer = [slot_size_bytes]
|
||||
self.block_len_per_layer = [slot_size_bytes * self.block_size]
|
||||
self.num_blocks = self.kv_cache_config.num_blocks
|
||||
self.num_regions = 1
|
||||
self.block_stride_per_layer = list(self.block_len_per_layer)
|
||||
self.region_num_blocks = [self.num_blocks]
|
||||
@@ -635,6 +639,12 @@ class TestNixlHandshake:
|
||||
if pcp_rank > 0:
|
||||
assert connector.get_finished(set()) == (set(), set())
|
||||
|
||||
worker.get_transfer_results = MagicMock(
|
||||
return_value=KVConnectorTransferResults(finished_sending={"sent"})
|
||||
)
|
||||
results = connector.get_transfer_results(set())
|
||||
assert results.finished_sending == ({"sent"} if pcp_rank == 0 else set())
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
@@ -1195,6 +1205,7 @@ class TestNixlHandshake:
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name=worker.backend_name,
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
region_num_blocks=None,
|
||||
)
|
||||
|
||||
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
|
||||
@@ -1435,9 +1446,9 @@ def test_kv_connector_stats(default_vllm_config, dist_init):
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=([1, 2, 3],),
|
||||
local_block_ids=([0],),
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": ([4, 5, 6],),
|
||||
"remote_block_ids": ([0],),
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
@@ -1482,6 +1493,10 @@ def test_kv_connector_stats(default_vllm_config, dist_init):
|
||||
assert stats_after_reset is None
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_reqs_to_send_deadline_rebased_to_worker_clock(default_vllm_config, dist_init):
|
||||
"""reqs_to_send deadlines are stamped with the scheduler process's
|
||||
perf_counter, whose epoch differs across processes and (by boot-time
|
||||
@@ -1906,9 +1921,10 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
|
||||
assert handle == 22
|
||||
assert worker._dram_src_handles_by_block_size[worker.block_size] == 11
|
||||
assert [block[2] for block in blocks] == [0, 0, 3, 3]
|
||||
assert [
|
||||
memory_types = [
|
||||
call.args[1] for call in worker.nixl_wrapper.get_xfer_descs.call_args_list
|
||||
] == ["DRAM", "VRAM"]
|
||||
]
|
||||
assert memory_types == ["DRAM", "VRAM"]
|
||||
|
||||
|
||||
def test_mixed_memory_read_notifies_after_both_transfers_finish():
|
||||
@@ -1919,6 +1935,7 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish():
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._pending_recv_notifs = {}
|
||||
worker._failed_inflight_recvs = set()
|
||||
worker.xfer_stats = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
|
||||
@@ -1959,6 +1976,7 @@ def test_mixed_memory_read_failure_does_not_notify_producer():
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker._recving_transfers = {"request": [101, 102]}
|
||||
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
|
||||
worker._failed_inflight_recvs = set()
|
||||
worker._is_hma_required = True
|
||||
worker._failed_recv_reqs = MagicMock()
|
||||
worker._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
@@ -2871,9 +2889,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
)
|
||||
def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init):
|
||||
"""Test that transfer setup failures mark blocks invalid
|
||||
and return via get_finished."""
|
||||
@pytest.mark.parametrize("is_hma", [False, True])
|
||||
def test_transfer_setup_failure_returns_finished(
|
||||
default_vllm_config, dist_init, is_hma
|
||||
):
|
||||
"""Setup failures report the request; only non-HMA reports block IDs."""
|
||||
vllm_config = create_vllm_config()
|
||||
|
||||
connector = NixlConnector(
|
||||
@@ -2883,6 +2903,7 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init)
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
)
|
||||
connector.connector_worker.nixl_wrapper.fail_transfer_setup = True
|
||||
connector.connector_worker._is_hma_required = is_hma
|
||||
|
||||
request_id = "test_transfer_fail"
|
||||
metadata = NixlConnectorMetadata()
|
||||
@@ -2912,13 +2933,106 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init)
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
# check that blocks were marked invalid
|
||||
results = connector.get_transfer_results(finished_req_ids=set())
|
||||
assert request_id in results.finished_recving
|
||||
assert results.failed_recving == {request_id}
|
||||
invalid_blocks = connector.get_block_ids_with_load_errors()
|
||||
assert invalid_blocks == {7, 8, 9}
|
||||
assert invalid_blocks == (set() if is_hma else {7, 8, 9})
|
||||
|
||||
# ensure request appears in get_finished
|
||||
|
||||
class _ScriptedXferWrapper(FakeNixlWrapper):
|
||||
"""Scripts per-handle xfer states; forbids releasing in-flight handles."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# handle -> successive states; the last one repeats.
|
||||
self.xfer_states: dict[int, list[str]] = {}
|
||||
self.released: list[int] = []
|
||||
|
||||
def check_xfer_state(self, handle: int) -> str:
|
||||
states = self.xfer_states[handle]
|
||||
return states.pop(0) if len(states) > 1 else states[0]
|
||||
|
||||
def release_xfer_handle(self, handle: int) -> None:
|
||||
# A posted-but-unfinished transfer cannot be aborted: releasing it
|
||||
# leaves the RDMA READ armed. Production code must never do this.
|
||||
assert self.xfer_states.get(handle, ["DONE"])[0] != "PROC", (
|
||||
f"released in-flight handle {handle}"
|
||||
)
|
||||
self.released.append(handle)
|
||||
|
||||
|
||||
def _make_split_read_connector(vllm_config, request_id, states):
|
||||
"""Seed a request with scripted in-flight xfer handles + recv metadata."""
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
wrapper = _ScriptedXferWrapper("agent")
|
||||
worker.nixl_wrapper = wrapper
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=([7, 8, 9],),
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": ([10, 11, 12],),
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"remote_tp_size": 1,
|
||||
},
|
||||
)
|
||||
worker._recving_metadata[request_id] = metadata.reqs_to_recv[request_id]
|
||||
wrapper.xfer_states = dict(states)
|
||||
worker._recving_transfers[request_id] = list(states)
|
||||
return connector, worker, wrapper
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_split_read_failure_defers_report_until_last_handle(
|
||||
default_vllm_config, dist_init
|
||||
):
|
||||
"""One half of a split (mixed DRAM/VRAM) read failing must not report the
|
||||
request — nor invalidate its blocks — while the sibling xfer is still in
|
||||
flight: a posted READ cannot be aborted and would DMA into blocks the
|
||||
scheduler could free and reuse. The report happens exactly once, when the
|
||||
last handle is terminal."""
|
||||
request_id = "split_read_partial_failure"
|
||||
err_handle, live_handle = 11, 22
|
||||
connector, worker, wrapper = _make_split_read_connector(
|
||||
create_vllm_config(),
|
||||
request_id,
|
||||
{err_handle: ["ERR"], live_handle: ["PROC", "PROC", "DONE"]},
|
||||
)
|
||||
|
||||
# Poll 1: one half fails; the sibling is in flight -> nothing reported.
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert request_id in done_recving
|
||||
assert done_recving == set()
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
assert request_id in worker._recving_metadata
|
||||
assert wrapper.released == [err_handle]
|
||||
|
||||
# Poll 2: sibling still in flight.
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert done_recving == set()
|
||||
|
||||
# Poll 3: sibling terminal -> reported exactly once, blocks invalidated.
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert done_recving == {request_id}
|
||||
assert connector.get_block_ids_with_load_errors() == {7, 8, 9}
|
||||
assert request_id not in worker._recving_metadata
|
||||
assert wrapper.released == [err_handle, live_handle]
|
||||
|
||||
# Poll 4: nothing left; no double report.
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert done_recving == set()
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -2938,7 +3052,7 @@ def test_failed_request_skips_kv_postprocessing(
|
||||
default_vllm_config, dist_init, failure_mode
|
||||
):
|
||||
"""Test that failed requests skip KV sync and post-processing in
|
||||
get_finished().
|
||||
get_transfer_results().
|
||||
|
||||
This is the core safety behavior: when a KV transfer fails at any stage,
|
||||
the request must still appear in done_recving (so the scheduler can apply
|
||||
@@ -3023,11 +3137,10 @@ def test_failed_request_skips_kv_postprocessing(
|
||||
patch.object(worker, "sync_recved_kv_to_device") as mock_sync,
|
||||
patch.object(worker, "post_process_device_kv_on_receive") as mock_postprocess,
|
||||
):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
results = connector.get_transfer_results(finished_req_ids=set())
|
||||
|
||||
# The failed request must appear in done_recving so the scheduler
|
||||
# can handle it (e.g., trigger recompute via kv_load_failure_policy).
|
||||
assert request_id in done_recving
|
||||
assert request_id in results.finished_recving
|
||||
assert results.failed_recving == {request_id}
|
||||
|
||||
# Critical: KV sync and post-processing must NOT have been called
|
||||
# since no valid KV data was received for the failed request.
|
||||
@@ -3049,19 +3162,7 @@ def test_failed_request_skips_kv_postprocessing(
|
||||
def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
|
||||
default_vllm_config, dist_init
|
||||
):
|
||||
"""A request whose handles fail in different polls must not crash the engine.
|
||||
|
||||
One transfer handle is created per remote rank, and when a peer goes away
|
||||
they do not all fail in the same poll: one errors while another is still
|
||||
PROC. The first failure reports the request via _failed_recv_reqs and
|
||||
get_finished() pops its metadata; when the remaining handles fail in a
|
||||
later poll, the request must be cleaned up without being reported again.
|
||||
|
||||
Reporting twice kills the EngineCore: the scheduler's assert in
|
||||
_update_from_kv_xfer_finished only expects a finished recv for a request
|
||||
still waiting for KVs, having moved this one out of
|
||||
WAITING_FOR_REMOTE_KVS to recompute locally after the first report.
|
||||
"""
|
||||
"""Failures across polls report once, after the last writer is terminal."""
|
||||
vllm_config = create_vllm_config()
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
@@ -3107,20 +3208,21 @@ def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
|
||||
):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
|
||||
assert request_id in done_recving
|
||||
assert request_id not in worker._recving_metadata
|
||||
assert request_id not in done_recving
|
||||
assert request_id in worker._recving_metadata
|
||||
assert worker._recving_transfers[request_id] == [second]
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
|
||||
# Poll 2: the second handle fails too. The request was already reported,
|
||||
# so it must not be reported a second time: the scheduler has since moved
|
||||
# it out of WAITING_FOR_REMOTE_KVS to recompute locally, and asserts on a
|
||||
# finished recv for a request in that state. Its remaining handles are
|
||||
# still released and the transfer entry removed.
|
||||
# Only the last terminal handle permits the scheduler to reuse the blocks.
|
||||
with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
|
||||
assert request_id not in done_recving
|
||||
assert done_recving == {request_id}
|
||||
assert request_id not in worker._recving_transfers
|
||||
assert request_id not in worker._recving_metadata
|
||||
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert done_recving == set()
|
||||
|
||||
|
||||
def _set_test_speculative_config(
|
||||
|
||||
@@ -5,16 +5,31 @@
|
||||
import gc
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import MockMambaBuilder
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.config import KVTransferConfig, set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
FullAttentionManager,
|
||||
SlidingWindowManager,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
from .utils import (
|
||||
create_request,
|
||||
@@ -272,26 +287,24 @@ def test_logical_to_kernel_block_ids_with_hma():
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"is_rocm,has_mamba,use_host_buffer,done_recving,failed_recving,expected_syncs",
|
||||
"is_rocm,has_mamba,use_host_buffer,done_recving,expected_syncs",
|
||||
[
|
||||
(True, True, False, {"req"}, set(), 1),
|
||||
(False, True, False, {"req"}, set(), 0),
|
||||
(True, False, False, {"req"}, set(), 0),
|
||||
(True, True, True, {"req"}, set(), 0),
|
||||
(True, True, False, set(), set(), 0),
|
||||
(True, True, False, {"req"}, {"req"}, 0),
|
||||
(True, True, False, {"req"}, 1),
|
||||
(False, False, False, {"req"}, 0),
|
||||
(False, True, False, {"req"}, 0),
|
||||
(True, True, True, {"req"}, 0),
|
||||
(True, True, False, set(), 0),
|
||||
],
|
||||
)
|
||||
def test_sync_device_after_mamba_recv_gates(
|
||||
def test_sync_device_after_direct_recv_gates(
|
||||
monkeypatch,
|
||||
is_rocm,
|
||||
has_mamba,
|
||||
use_host_buffer,
|
||||
done_recving,
|
||||
failed_recving,
|
||||
expected_syncs,
|
||||
):
|
||||
"""Only direct-GPU Mamba receives on ROCm need a device fence."""
|
||||
"""Only backends requiring visibility fences synchronize direct receives."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
@@ -309,7 +322,7 @@ def test_sync_device_after_mamba_recv_gates(
|
||||
lambda: sync_calls.append(True),
|
||||
)
|
||||
|
||||
worker._sync_device_after_mamba_recv(done_recving, failed_recving)
|
||||
worker._sync_device_after_direct_recv(done_recving)
|
||||
|
||||
assert len(sync_calls) == expected_syncs
|
||||
|
||||
@@ -399,6 +412,8 @@ def test_read_blocks_for_req_expands_remote_ids(
|
||||
)
|
||||
|
||||
remote_engine_id = "remote-engine"
|
||||
worker.region_group_ids = list(range(len(resolved_types)))
|
||||
worker.dst_region_group_ids = {remote_engine_id: list(worker.region_group_ids)}
|
||||
|
||||
worker.transfer_topo = MagicMock()
|
||||
# tp_ratio not exercised (all_source_ranks is empty so no reads run),
|
||||
@@ -888,6 +903,45 @@ def _make_mock_worker_for_desc_ids(
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_shared_nixl_region_broadcasts_every_group_block_table():
|
||||
"""A shared region must receive every group's block table exactly once."""
|
||||
worker = _make_mock_worker_for_desc_ids(
|
||||
num_regions=3,
|
||||
has_mamba=False,
|
||||
group_spec_types=(FullAttentionSpec,) * 3,
|
||||
block_len_per_layer=[100] * 3,
|
||||
)
|
||||
worker.region_group_ids = [0, bw._SHARED_REGION_GROUP_ID, 2]
|
||||
worker._uses_region_group_mapping = True
|
||||
|
||||
result = worker._compute_desc_ids(
|
||||
block_ids=([1], [2], [3]),
|
||||
dst_num_blocks=10,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
region_num_blocks=[10, 10, 10],
|
||||
)
|
||||
|
||||
assert result.tolist() == [1, 11, 12, 13, 23]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_nixl_region_sort_preserves_pipeline_layer_order():
|
||||
"""Numeric layer order prevents PP peers from pairing different regions."""
|
||||
names = [
|
||||
"model.layers.10.self_attn",
|
||||
"model.layers.2.self_attn",
|
||||
"model.layers.1.self_attn",
|
||||
]
|
||||
|
||||
assert sorted(names, key=bw._region_sort_key) == [
|
||||
"model.layers.1.self_attn",
|
||||
"model.layers.2.self_attn",
|
||||
"model.layers.10.self_attn",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_get_block_descs_ids_hybrid_ssm():
|
||||
"""Test _compute_desc_ids uses per-group strides for hybrid
|
||||
@@ -1097,10 +1151,7 @@ def test_post_process_zeroes_untransferred_tail():
|
||||
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
|
||||
fa_group = MagicMock(layer_names=["attn.0"])
|
||||
ssm_group = MagicMock(layer_names=["mamba.0"])
|
||||
worker.kv_cache_config = MagicMock(
|
||||
kv_cache_groups=[fa_group, ssm_group],
|
||||
transfer_groups=[fa_group, ssm_group],
|
||||
)
|
||||
worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, ssm_group])
|
||||
# The cached property filters mamba layers out of the permuted caches.
|
||||
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
|
||||
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
|
||||
@@ -1188,6 +1239,11 @@ def _make_fake_kv_cache_manager():
|
||||
|
||||
manager = object.__new__(KVCacheManager)
|
||||
manager.coordinator = MagicMock()
|
||||
manager.kv_cache_config = MagicMock()
|
||||
manager.kv_cache_config.kv_cache_groups = (
|
||||
MagicMock(host_resident=False),
|
||||
MagicMock(host_resident=False),
|
||||
)
|
||||
manager.coordinator.single_type_managers = (
|
||||
_FakeSingleTypeManager(True, 16, [10, 11, 12, 13, 14, 15]), # attention
|
||||
_FakeSingleTypeManager(False, 16, [20, 21, 22, 23, 24, 25]), # mamba
|
||||
@@ -1723,16 +1779,6 @@ def test_exchange_clipped_blocks_ssm_positional_states():
|
||||
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_excludes_nontransfer_groups():
|
||||
"""Scheduler block tables must match the worker's registered regions."""
|
||||
sched = make_nixl_scheduler()
|
||||
sched.kv_cache_config = make_kv_cache_config(block_size=16, swa_enabled=True)
|
||||
sched.kv_cache_config.kv_cache_groups[1].enable_kv_transfer = False
|
||||
|
||||
assert sched.get_exchange_clipped_blocks(([1, 2], [9])) == ([1, 2],)
|
||||
|
||||
|
||||
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
|
||||
|
||||
|
||||
@@ -1786,6 +1832,95 @@ def _make_hybrid_mla_kv_cache_config(num_blocks: int = 4):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("kernel_block_size", [16, 8])
|
||||
def test_nixl_keeps_device_block_count_with_hisparse_host_pool(kernel_block_size):
|
||||
host_num_blocks = 4
|
||||
gpu_num_blocks = 9
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=gpu_num_blocks,
|
||||
hisparse_host_num_blocks=host_num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=host_num_blocks * spec.page_size_bytes,
|
||||
layers=["mla.host"],
|
||||
layer_stride=host_num_blocks * spec.page_size_bytes,
|
||||
block_stride=spec.page_size_bytes,
|
||||
host_resident=True,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=gpu_num_blocks * spec.page_size_bytes,
|
||||
layers=["mla.device"],
|
||||
layer_stride=gpu_num_blocks * spec.page_size_bytes,
|
||||
block_stride=spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["mla.host"],
|
||||
spec,
|
||||
host_resident=True,
|
||||
),
|
||||
KVCacheGroupSpec(["mla.device"], spec),
|
||||
],
|
||||
)
|
||||
vllm_config = create_vllm_config(block_size=16)
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [kernel_block_size]
|
||||
fake_backend.get_name.return_value = "FLASHMLA"
|
||||
fake_backend.full_cls_name.return_value = "fake.FLASHMLA"
|
||||
fake_backend.get_kv_cache_shape.return_value = (1, kernel_block_size, 1, 1)
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
physical_per_logical = spec.block_size // kernel_block_size
|
||||
physical_page_bytes = spec.page_size_bytes // physical_per_logical
|
||||
host_cache = torch.zeros(
|
||||
host_num_blocks * physical_per_logical, physical_page_bytes, dtype=torch.uint8
|
||||
)
|
||||
device_cache = torch.zeros(
|
||||
gpu_num_blocks * physical_per_logical, physical_page_bytes, dtype=torch.uint8
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper"),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "test-engine", kv_cache_config)
|
||||
worker.use_mla = True
|
||||
worker.nixl_wrapper.get_agent_metadata.return_value = b"metadata"
|
||||
worker.register_kv_caches({"mla.host": host_cache, "mla.device": device_cache})
|
||||
|
||||
assert worker.num_blocks == gpu_num_blocks * physical_per_logical
|
||||
assert dict(zip(worker.region_group_ids, worker.region_num_blocks)) == {
|
||||
0: host_num_blocks * physical_per_logical,
|
||||
1: gpu_num_blocks * physical_per_logical,
|
||||
}
|
||||
caches = {"mla.host": host_cache, "mla.device": device_cache}
|
||||
expected_addrs = [
|
||||
caches[name].data_ptr() + block * stride
|
||||
for name, stride, count in zip(
|
||||
worker.region_names,
|
||||
worker.block_stride_per_layer,
|
||||
worker.region_num_blocks,
|
||||
strict=True,
|
||||
)
|
||||
for block in range(count)
|
||||
]
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
"""Hybrid MLA+KDA registration: HMA tensors shared by both layer types
|
||||
@@ -1803,6 +1938,9 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
kv_cache_config = _make_hybrid_mla_kv_cache_config()
|
||||
unified_page = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
vllm_config = create_vllm_config(block_size=12)
|
||||
# The engine rewrites this global value to the smallest scheduler-group
|
||||
# block size. NIXL must retain the transferable groups' 12-token geometry.
|
||||
vllm_config.cache_config.block_size = 4
|
||||
# kv_buffer_device defaults to the *real* platform's device type, which on
|
||||
# a CPU-only test host would make this a host-buffer worker: host xfer
|
||||
# buffers are per-layer, so the HMA shared tensors would not be
|
||||
@@ -1833,7 +1971,13 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
worker.use_mla = True # opt-125m test config is not MLA; force the flag
|
||||
worker.nixl_wrapper.get_agent_metadata.return_value = b"fake-agent-metadata"
|
||||
|
||||
tensors = [torch.zeros(4 * unified_page, dtype=torch.uint8) for _ in range(2)]
|
||||
kernel_page = unified_page // 3
|
||||
block_stride = 2 * kernel_page
|
||||
backing = torch.zeros(12, block_stride, dtype=torch.uint8)
|
||||
tensors = [
|
||||
backing[:, :kernel_page],
|
||||
backing[:, kernel_page:],
|
||||
]
|
||||
# KDA layer first per tensor: exercises the dual-purpose flag merge.
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
@@ -1849,6 +1993,8 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
# 12-token logical blocks over the 4-token MLA kernel block.
|
||||
assert worker._physical_blocks_per_logical_kv_block == 3
|
||||
assert worker.block_size == 4 and worker.num_blocks == 12
|
||||
assert worker.region_group_ids == [bw._SHARED_REGION_GROUP_ID] * 2
|
||||
assert worker.block_stride_per_layer == [block_stride, block_stride]
|
||||
# Both shared tensors are dual-purpose: their FA view is MLA even though
|
||||
# a KDA layer registered them first.
|
||||
assert worker._region_is_mla == [True, True]
|
||||
@@ -1861,8 +2007,20 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
# Mamba descs: 2 regions x (3 conv sub-projections + 1 ssm) x 4 blocks.
|
||||
assert worker.src_blocks_data.shape == (24 + 32, 3)
|
||||
fa_descs = worker.src_blocks_data[:24]
|
||||
assert fa_descs[1][0] - fa_descs[0][0] == unified_page // 3
|
||||
expected_addrs = [
|
||||
tensor.data_ptr() + block * block_stride
|
||||
for tensor in tensors
|
||||
for block in range(12)
|
||||
]
|
||||
assert fa_descs[:, 0].tolist() == expected_addrs
|
||||
assert all(size == unified_page // 3 for size in fa_descs[:, 1])
|
||||
worker.nixl_wrapper.register_memory.assert_called_once()
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes,
|
||||
type=NixlAgentMetadata,
|
||||
)
|
||||
assert metadata.block_strides == [block_stride, block_stride]
|
||||
assert metadata.region_num_blocks == [12, 12]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@@ -1909,6 +2067,8 @@ def test_push_write_hybrid_mla_replicates_attention():
|
||||
worker.dst_xfer_side_handles = {engine_id: {0: 100, 1: 101}}
|
||||
worker.src_xfer_handles_by_tp_ratio = {(-2, 4): [200, 201]}
|
||||
worker.src_xfer_handles_by_block_size = {4: 300}
|
||||
worker.region_group_ids = [0, 1]
|
||||
worker.dst_region_group_ids = {engine_id: [0, 1]}
|
||||
worker._sending_transfers = defaultdict(list)
|
||||
worker._sending_transfers_lock = threading.Lock()
|
||||
worker.kv_cache_config = _make_hybrid_mla_kv_cache_config()
|
||||
|
||||
@@ -186,11 +186,16 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
worker.host_buffer_kv_cache_layout = "NHD"
|
||||
worker._physical_blocks_per_logical_kv_block = 1
|
||||
worker._logical_num_blocks = num_blocks
|
||||
worker.region_mem_types = []
|
||||
worker.region_group_ids = []
|
||||
worker.region_mem_types = []
|
||||
worker._mixed_mem_types = False
|
||||
worker.region_names = []
|
||||
worker.region_num_blocks = []
|
||||
worker._mixed_mem_types = False
|
||||
worker._desc_is_dram_by_block_size = {}
|
||||
worker._desc_pos_by_block_size = {}
|
||||
worker._dram_src_handles_by_block_size = {}
|
||||
worker._region_is_mla = []
|
||||
worker.block_len_per_layer = []
|
||||
worker.block_stride_per_layer = []
|
||||
@@ -304,8 +309,6 @@ def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blo
|
||||
# would not be deduplicated. Pin it to the faked device type.
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [kernel_block_size]
|
||||
fake_backend.get_name.return_value = "FLASHMLA"
|
||||
@@ -552,6 +555,16 @@ def _make_remote_meta(
|
||||
)
|
||||
|
||||
|
||||
def _register_remote_agents(worker, metadata, tp_size):
|
||||
"""Mirror the async handshake callback that publishes prepared agents."""
|
||||
worker._remote_agents[metadata.engine_id] = {
|
||||
(0, rank): worker.add_remote_agent(
|
||||
metadata, remote_tp_rank=rank, remote_tp_size=tp_size
|
||||
)
|
||||
for rank in range(tp_size)
|
||||
}
|
||||
|
||||
|
||||
def _owned_byte_ranges(worker, group_logical_ids):
|
||||
"""Byte ranges owned by a request: for each HMA region tensor, every
|
||||
logical block id of every group maps to one unified page."""
|
||||
@@ -609,8 +622,7 @@ def test_hetero_ppl_multi_read_writes_stay_within_request_blocks():
|
||||
remote_num_logical=12,
|
||||
remote_ssm_sizes=(24, 32),
|
||||
)
|
||||
for rank in (0, 1):
|
||||
worker.add_remote_agent(meta_r, remote_tp_rank=rank, remote_tp_size=2)
|
||||
_register_remote_agents(worker, meta_r, 2)
|
||||
|
||||
# Request B: 17 matched tokens. Local: 2 logical blocks (24 tok
|
||||
# capacity); remote: 16 prefilled tokens -> 2 remote logical blocks.
|
||||
@@ -626,7 +638,7 @@ def test_hetero_ppl_multi_read_writes_stay_within_request_blocks():
|
||||
"remote_block_ids": remote_ids,
|
||||
"remote_engine_id": "remote-engine",
|
||||
"remote_request_id": "prefill-req-b",
|
||||
"remote_host": "localhost",
|
||||
"remote_host": "remote-host",
|
||||
"remote_port": 1234,
|
||||
"tp_size": 2,
|
||||
},
|
||||
@@ -709,8 +721,7 @@ def _run_hetero_case(
|
||||
remote_num_logical=max(2 * n_remote + 4, 8),
|
||||
remote_ssm_sizes=(48 // tp_size, 64 // tp_size),
|
||||
)
|
||||
for rank in range(tp_size):
|
||||
worker.add_remote_agent(meta_r, remote_tp_rank=rank, remote_tp_size=tp_size)
|
||||
_register_remote_agents(worker, meta_r, tp_size)
|
||||
|
||||
# Sparse ids so neighbors exist between the request's blocks.
|
||||
local_attn = [2 * i + 1 for i in range(n_local)]
|
||||
@@ -726,7 +737,7 @@ def _run_hetero_case(
|
||||
"remote_block_ids": remote_ids,
|
||||
"remote_engine_id": "remote-engine",
|
||||
"remote_request_id": "prefill-req-b",
|
||||
"remote_host": "localhost",
|
||||
"remote_host": "remote-host",
|
||||
"remote_port": 1234,
|
||||
"tp_size": tp_size,
|
||||
},
|
||||
|
||||
@@ -15,7 +15,7 @@ requiring a real NIXL agent or network:
|
||||
* The worker matches D registrations against P finished blocks (both
|
||||
scenario directions) and forwards non-PUSH_REG NIXL notifs to the main
|
||||
thread's ``_get_new_notifs``.
|
||||
* ``get_finished`` enqueues evictions for the writer.
|
||||
* ``get_transfer_results`` enqueues evictions for the writer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,6 +32,9 @@ from unittest.mock import MagicMock, patch
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
PUSH_REG_NOTIF_PREFIX,
|
||||
NixlAgentMetadata,
|
||||
@@ -347,6 +350,8 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
|
||||
# Base worker fields touched by start_load_kv / _get_new_notifs.
|
||||
w._recving_metadata = {}
|
||||
w._pending_recv_notifs = {}
|
||||
w._failed_inflight_recvs = set()
|
||||
w._recving_transfers = defaultdict(list)
|
||||
w._reqs_to_process = set()
|
||||
w._reqs_to_send = {}
|
||||
@@ -742,21 +747,19 @@ class TestPushWriterNotifs:
|
||||
assert notified == set()
|
||||
assert request_id in w._recving_transfers
|
||||
|
||||
def test_get_finished_evicts_completed_state(self):
|
||||
"""``get_finished`` should enqueue evictions and wake the writer."""
|
||||
def test_get_transfer_results_evicts_completed_state(self):
|
||||
"""Transfer completion should enqueue evictions and wake the writer."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
|
||||
# Stub the base ``get_finished`` to return one done_sending entry.
|
||||
# Patch via the MRO's parent class.
|
||||
with patch.object(
|
||||
NixlPushConnectorWorker.__mro__[1],
|
||||
"get_finished",
|
||||
return_value=({"req-done"}, set()),
|
||||
"get_transfer_results",
|
||||
return_value=KVConnectorTransferResults(finished_sending={"req-done"}),
|
||||
):
|
||||
done_sending, done_recving = w.get_finished()
|
||||
results = w.get_transfer_results()
|
||||
|
||||
assert "req-done" in done_sending
|
||||
assert done_recving == set()
|
||||
assert results.finished_sending == {"req-done"}
|
||||
assert results.finished_recving == set()
|
||||
# Eviction enqueued for the writer.
|
||||
evicted = []
|
||||
while True:
|
||||
@@ -928,19 +931,21 @@ class TestPushWriterNegative:
|
||||
assert len(w._pending_d_registrations) == 1
|
||||
assert w.start_push_calls == []
|
||||
|
||||
def test_get_finished_enqueues_eviction_for_each_done_request(self):
|
||||
"""``get_finished`` must enqueue an eviction for every request
|
||||
def test_get_transfer_results_enqueues_each_completed_request(self):
|
||||
"""``get_transfer_results`` must enqueue every completed request
|
||||
in ``done_sending`` so the writer can drop stale matching state.
|
||||
Unlike the happy-path test, this verifies the *cardinality*: N
|
||||
completed requests -> N evictions, in order."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
with patch.object(
|
||||
NixlPushConnectorWorker.__mro__[1],
|
||||
"get_finished",
|
||||
return_value=({"req-1", "req-2", "req-3"}, set()),
|
||||
"get_transfer_results",
|
||||
return_value=KVConnectorTransferResults(
|
||||
finished_sending={"req-1", "req-2", "req-3"}
|
||||
),
|
||||
):
|
||||
done_sending, _ = w.get_finished()
|
||||
assert done_sending == {"req-1", "req-2", "req-3"}
|
||||
results = w.get_transfer_results()
|
||||
assert results.finished_sending == {"req-1", "req-2", "req-3"}
|
||||
|
||||
evicted: list[str] = []
|
||||
while True:
|
||||
@@ -950,23 +955,47 @@ class TestPushWriterNegative:
|
||||
break
|
||||
assert sorted(evicted) == ["req-1", "req-2", "req-3"]
|
||||
|
||||
def test_get_finished_with_no_completions_does_not_enqueue_eviction(self):
|
||||
def test_empty_transfer_results_do_not_enqueue_eviction(self):
|
||||
"""If there's nothing newly done, no eviction should be enqueued.
|
||||
The wake event IS still set because ``get_finished`` always wakes
|
||||
the writer to drain notifs."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
with patch.object(
|
||||
NixlPushConnectorWorker.__mro__[1],
|
||||
"get_finished",
|
||||
return_value=(set(), set()),
|
||||
"get_transfer_results",
|
||||
return_value=KVConnectorTransferResults(),
|
||||
):
|
||||
done_sending, done_recving = w.get_finished()
|
||||
assert done_sending == set()
|
||||
assert done_recving == set()
|
||||
results = w.get_transfer_results()
|
||||
assert results.finished_sending == set()
|
||||
assert results.finished_recving == set()
|
||||
assert w._evict_finished_inbox.qsize() == 0
|
||||
# Wake set so the writer drains NIXL notifs even when idle.
|
||||
assert w._push_writer_wake.is_set()
|
||||
|
||||
def test_failed_push_transfer_finishes_sending(self):
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.nixl_wrapper.check_xfer_state.return_value = "ERR"
|
||||
w.xfer_stats = MagicMock()
|
||||
w._log_failure = MagicMock()
|
||||
w._sending_transfers["req-failed"] = [17]
|
||||
w._reqs_to_send["req-failed"] = time.perf_counter() + 30
|
||||
w._reqs_to_process.add("req-failed")
|
||||
|
||||
with patch.object(
|
||||
NixlPushConnectorWorker.__mro__[1],
|
||||
"get_transfer_results",
|
||||
return_value=KVConnectorTransferResults(),
|
||||
):
|
||||
results = w.get_transfer_results()
|
||||
|
||||
assert results.finished_sending == {"req-failed"}
|
||||
assert results.finished_recving == set()
|
||||
assert "req-failed" not in w._sending_transfers
|
||||
assert "req-failed" not in w._reqs_to_send
|
||||
assert "req-failed" not in w._reqs_to_process
|
||||
w.nixl_wrapper.release_xfer_handle.assert_called_once_with(17)
|
||||
|
||||
def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog):
|
||||
"""A completion notif for a request the worker doesn't know
|
||||
about should be logged but not crash."""
|
||||
@@ -1160,6 +1189,7 @@ class TestPushWriterMlaReplication:
|
||||
)
|
||||
}
|
||||
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
|
||||
w.dst_region_group_ids[engine_id] = [0]
|
||||
w.dst_xfer_side_handles = {engine_id: {r: 1000 + r for r in d_ranks}}
|
||||
w.src_xfer_handles_by_block_size = {16: 2000}
|
||||
w._remote_agents = {engine_id: {(0, r): f"agent-{r}" for r in d_ranks}}
|
||||
@@ -1240,6 +1270,8 @@ class TestPushWriterMlaReplication:
|
||||
)
|
||||
}
|
||||
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
|
||||
w.region_group_ids = [0, 1, 2, 3]
|
||||
w.dst_region_group_ids[engine_id] = [0, 1, 2, 3]
|
||||
w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}}
|
||||
w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]}
|
||||
|
||||
|
||||
@@ -15,12 +15,14 @@ class DummyModelRunnerOutput(ModelRunnerOutput):
|
||||
finished_sending: set[str] | None = None,
|
||||
finished_recving: set[str] | None = None,
|
||||
invalid_block_ids: set[int] | None = None,
|
||||
failed_recving: set[str] | None = None,
|
||||
expected_finished_count: int = 0,
|
||||
):
|
||||
self.kv_connector_output = KVConnectorOutput(
|
||||
finished_sending=finished_sending,
|
||||
finished_recving=finished_recving,
|
||||
invalid_block_ids=invalid_block_ids or set(),
|
||||
failed_recving=failed_recving or set(),
|
||||
expected_finished_count=expected_finished_count,
|
||||
)
|
||||
|
||||
@@ -73,7 +75,9 @@ def test_aggregate_workers_output():
|
||||
|
||||
output1 = DummyModelRunnerOutput(invalid_block_ids={3, 4})
|
||||
output2 = DummyModelRunnerOutput(
|
||||
finished_recving={"req2"}, invalid_block_ids={4, 5}
|
||||
finished_recving={"req2"},
|
||||
invalid_block_ids={4, 5},
|
||||
failed_recving={"req3"},
|
||||
)
|
||||
|
||||
aggregated = aggregator.aggregate([output1, output2])
|
||||
@@ -83,6 +87,12 @@ def test_aggregate_workers_output():
|
||||
assert aggregated.finished_sending is None
|
||||
assert aggregated.finished_recving == {"req2"}
|
||||
assert aggregated.invalid_block_ids == {3, 4, 5}
|
||||
assert not aggregated.failed_recving
|
||||
|
||||
output1 = DummyModelRunnerOutput(finished_recving={"req3"})
|
||||
output2 = DummyModelRunnerOutput(finished_recving={"req3"})
|
||||
aggregated = aggregator.aggregate([output1, output2])
|
||||
assert aggregated.kv_connector_output.failed_recving == {"req3"}
|
||||
|
||||
|
||||
def test_aggregate_workers_output_with_expected_finished_count():
|
||||
|
||||
@@ -539,6 +539,7 @@ def make_nixl_scheduler(
|
||||
sched._heartbeat_interval = kv_lease_duration // 6
|
||||
# Fields touched by build_connector_meta / request_finished:
|
||||
sched._reqs_need_recv = {}
|
||||
sched._hisparse_host_blocks_to_recv = {}
|
||||
sched._reqs_need_send = {}
|
||||
sched._reqs_in_batch = set()
|
||||
sched._reqs_not_processed = set()
|
||||
|
||||
@@ -62,7 +62,7 @@ def _make_offloading_config(
|
||||
normalized_extra_config["cpu_bytes_to_use"] = cpu_bytes_to_use
|
||||
|
||||
if groups is None:
|
||||
groups = (OffloadingGroupConfig(16, ("layer",)),)
|
||||
groups = (OffloadingGroupConfig(16, ("layer",), 0),)
|
||||
|
||||
return OffloadingConfig(
|
||||
groups=groups,
|
||||
@@ -493,8 +493,8 @@ def test_offloading_spec_has_replicated_layout_default():
|
||||
|
||||
def test_offloading_spec_uses_normalized_chunk_geometry():
|
||||
groups = (
|
||||
OffloadingGroupConfig(12, ("full_layer",)),
|
||||
OffloadingGroupConfig(16, ("mla_layer",)),
|
||||
OffloadingGroupConfig(12, ("full_layer",), 0),
|
||||
OffloadingGroupConfig(16, ("mla_layer",), 1),
|
||||
)
|
||||
spec = _create_spec(
|
||||
groups=groups,
|
||||
|
||||
@@ -26,8 +26,11 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
OffloadingGroupConfig(
|
||||
tokens_per_block=tokens_per_block,
|
||||
layer_names=(layer_name,),
|
||||
group_id=group_id,
|
||||
)
|
||||
for group_id, (tokens_per_block, layer_name) in enumerate(
|
||||
kwargs.get("groups", ())
|
||||
)
|
||||
for tokens_per_block, layer_name in kwargs.get("groups", ())
|
||||
),
|
||||
worker_kv_bytes_per_block=0,
|
||||
enable_kv_cache_events=False,
|
||||
|
||||
@@ -400,6 +400,8 @@ def test_register_mixed_page_sizes_in_one_cache_group(monkeypatch):
|
||||
vllm_config.cache_config = CacheConfig()
|
||||
vllm_config.cache_config.kv_cache_layout = layout.name
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
vllm_config.attention_config.hisparse_config = None
|
||||
vllm_config.kv_transfer_config = None
|
||||
|
||||
pages = [spec.page_size_bytes for spec in specs.values()]
|
||||
num_blocks = 4
|
||||
@@ -458,6 +460,8 @@ def test_register_mixed_page_sizes_odd_block_counts(monkeypatch, rank_blocks):
|
||||
vllm_config.cache_config = CacheConfig()
|
||||
vllm_config.cache_config.kv_cache_layout = layout.name
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
vllm_config.attention_config.hisparse_config = None
|
||||
vllm_config.kv_transfer_config = None
|
||||
|
||||
pages = [spec.page_size_bytes for spec in specs.values()]
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
@@ -507,6 +511,8 @@ def test_mixed_page_byte_placement_is_dcp_invariant():
|
||||
vllm_config.cache_config.kv_cache_layout = KVCacheLayout.LBNHC.name
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
vllm_config.attention_config.hisparse_config = None
|
||||
vllm_config.kv_transfer_config = None
|
||||
|
||||
page_bytes = sum(spec.page_size_bytes for spec in specs.values())
|
||||
config = get_kv_cache_config_from_groups(vllm_config, [group], page_bytes * 4)
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CrossAttentionManager,
|
||||
FullAttentionManager,
|
||||
HiSparseSourceManager,
|
||||
MambaManager,
|
||||
SingleTypeKVCacheManager,
|
||||
SinkFullAttentionManager,
|
||||
@@ -27,6 +28,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
CrossAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KVCacheGroupRole,
|
||||
KVCacheSpec,
|
||||
KVCacheSpecKind,
|
||||
MambaSpec,
|
||||
@@ -176,6 +178,19 @@ class TestKVCacheSpecRegistry:
|
||||
is spec_uniform_base_map[spec_cls]
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("role", list(KVCacheGroupRole))
|
||||
def test_mla_manager_selection_by_role(self, role):
|
||||
"""Only the source role overrides ordinary MLA manager selection."""
|
||||
expected = (
|
||||
HiSparseSourceManager
|
||||
if role == KVCacheGroupRole.HISPARSE_SOURCE
|
||||
else FullAttentionManager
|
||||
)
|
||||
assert (
|
||||
KVCacheSpecRegistry.get_manager_class(make_spec(MLAAttentionSpec), role)
|
||||
is expected
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("spec_cls", list(spec_manager_map))
|
||||
def test_custom_spec_register(self, spec_cls):
|
||||
"""A decorated custom spec resolves to the declared manager."""
|
||||
|
||||
@@ -13,15 +13,18 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiSparseResidentSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
SparseCacheRole,
|
||||
compute_layout_strides,
|
||||
)
|
||||
from vllm.v1.worker.gpu import attn_utils
|
||||
@@ -36,6 +39,60 @@ from vllm.v1.worker.utils import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enabled", "block_size", "main_sizes", "indexer_sizes", "expected"),
|
||||
[
|
||||
(True, 256, [64], [64], 64),
|
||||
(True, 64, [32, 64], [16, 32], 32),
|
||||
(True, 64, [MultipleOf(16)], [32], 32),
|
||||
(True, 64, [64], [32], None),
|
||||
(False, 256, [64], [64], 256),
|
||||
],
|
||||
)
|
||||
def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
monkeypatch, enabled, block_size, main_sizes, indexer_sizes, expected
|
||||
):
|
||||
"""Resolve shared MLA geometry before planning; leave other specs alone."""
|
||||
specs = {
|
||||
"main": MLAAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
|
||||
),
|
||||
"indexer": MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
"dense": FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
|
||||
),
|
||||
}
|
||||
layers = {}
|
||||
for name, sizes in zip(specs, [main_sizes, indexer_sizes, [block_size]]):
|
||||
backend = SimpleNamespace(
|
||||
customize_spec=AttentionBackend.customize_spec,
|
||||
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
|
||||
)
|
||||
layers[name] = SimpleNamespace(
|
||||
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
|
||||
get_attn_backend=lambda backend=backend: backend,
|
||||
)
|
||||
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=object() if enabled else None)
|
||||
)
|
||||
if expected is None:
|
||||
with pytest.raises(ValueError, match="supported by every sparse"):
|
||||
attn_utils.get_kv_cache_spec(config)
|
||||
return
|
||||
|
||||
resolved = attn_utils.get_kv_cache_spec(config)
|
||||
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
|
||||
assert resolved["dense"] is specs["dense"]
|
||||
assert all(spec.block_size == block_size for spec in specs.values())
|
||||
|
||||
|
||||
class _FakeMetadataBuilder:
|
||||
def __init__(self, support: AttentionCGSupport):
|
||||
self.support = support
|
||||
@@ -390,3 +447,71 @@ def test_copy_kv_cache_blocks_with_virtual_block_splitting(
|
||||
torch.testing.assert_close(
|
||||
cache[dst_start + physical_idx], expected[layer_idx][physical_idx]
|
||||
)
|
||||
|
||||
|
||||
def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
|
||||
"""Host tensors get their own backing; view-less specs keep the raw one."""
|
||||
spec = FullAttentionSpec(
|
||||
block_size=2, num_kv_heads=1, head_size=4, dtype=torch.float32
|
||||
)
|
||||
page = spec.page_size_bytes
|
||||
resident_spec = HiSparseResidentSpec(block_size=2, page_size=page)
|
||||
device_size = 4 * page
|
||||
config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
hisparse_host_num_blocks=3,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=3 * page,
|
||||
layers=["source"],
|
||||
layer_stride=3 * page,
|
||||
block_stride=page,
|
||||
host_resident=True,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=device_size,
|
||||
layers=["indexer"],
|
||||
layer_stride=device_size,
|
||||
block_stride=page,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=device_size,
|
||||
layers=["resident"],
|
||||
layer_stride=device_size,
|
||||
block_stride=page,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["source"], spec, host_resident=True),
|
||||
KVCacheGroupSpec(["indexer"], spec),
|
||||
KVCacheGroupSpec(["resident"], resident_spec),
|
||||
],
|
||||
)
|
||||
host_buffers: list[torch.Tensor] = []
|
||||
|
||||
def host_allocator(size: int) -> torch.Tensor:
|
||||
host_buffers.append(torch.zeros(size, dtype=torch.int8))
|
||||
return host_buffers[-1]
|
||||
|
||||
caches = allocate_hisparse_kv_caches(
|
||||
config,
|
||||
torch.device("cpu"),
|
||||
KVCacheLayout.LBHNC,
|
||||
[2, 2, 2],
|
||||
SimpleNamespace(allocate=host_allocator),
|
||||
)
|
||||
assert len(config.kv_cache_tensors) == 3
|
||||
|
||||
assert [buf.numel() for buf in host_buffers] == [3 * page]
|
||||
assert caches["source"].shape[0] == 3
|
||||
assert (
|
||||
caches["source"].untyped_storage().data_ptr()
|
||||
== host_buffers[0].untyped_storage().data_ptr()
|
||||
)
|
||||
assert caches["indexer"].shape[0] == 4
|
||||
backing = caches["resident"]
|
||||
assert backing.dtype == torch.int8 and backing.numel() >= device_size
|
||||
assert (
|
||||
backing.untyped_storage().data_ptr()
|
||||
== caches["indexer"].untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
@@ -134,6 +134,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
|
||||
vllm_config.cache_config = CacheConfig()
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
vllm_config.cache_config.kv_cache_layout = "BLHNC"
|
||||
vllm_config.attention_config.hisparse_config = None
|
||||
config = get_kv_cache_config_from_groups(vllm_config, groups, 8 * 1024 * 1024)
|
||||
views = allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout.BLHNC, None)
|
||||
buf_ptr = views["g1.big"].data_ptr()
|
||||
|
||||
@@ -351,6 +351,8 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.num_speculative_steps = 4
|
||||
speculator.current_draft_step = torch.tensor(0)
|
||||
speculator.slot_mapping_observer = None
|
||||
speculator.host_mirror_forward_observer = None
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
positions=torch.arange(2),
|
||||
query_start_loc=torch.arange(3),
|
||||
|
||||
@@ -5,9 +5,11 @@ from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.worker.gpu.kv_connector as kv_connector_module
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorTransferResults
|
||||
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
|
||||
|
||||
|
||||
@@ -18,9 +20,9 @@ def _make_connector(
|
||||
backend = Mock()
|
||||
backend.handle_preemptions.side_effect = lambda _: events.append("handle")
|
||||
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
|
||||
backend.start_load_kv.side_effect = lambda _: events.append("start")
|
||||
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
|
||||
backend.wait_for_save.side_effect = lambda: events.append("wait")
|
||||
backend.get_finished.side_effect = lambda _: (set(), set())
|
||||
backend.get_transfer_results.return_value = KVConnectorTransferResults()
|
||||
backend.get_block_ids_with_load_errors.return_value = set()
|
||||
backend.get_kv_connector_stats.return_value = None
|
||||
backend.get_kv_connector_kv_cache_events.return_value = None
|
||||
@@ -61,7 +63,15 @@ def test_load_start_phase(
|
||||
connector = _make_connector(monkeypatch, events)
|
||||
output = _scheduler_output(has_sync_kv_loads)
|
||||
|
||||
connector.pre_forward(output) # type: ignore[arg-type]
|
||||
request_indices = torch.tensor([3, 1])
|
||||
request_ids = ["first", "second"]
|
||||
attn_metadata = {"layer": object()}
|
||||
connector.pre_forward( # type: ignore[arg-type]
|
||||
output,
|
||||
request_state_indices=request_indices,
|
||||
request_ids=request_ids,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
assert events == (
|
||||
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
|
||||
)
|
||||
@@ -69,6 +79,16 @@ def test_load_start_phase(
|
||||
connector.post_forward(set())
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
|
||||
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
|
||||
assert kwargs["request_state_indices"] is request_indices
|
||||
assert kwargs["request_ids"] is request_ids
|
||||
assert kwargs["attn_metadata"] is attn_metadata
|
||||
|
||||
# A subsequent step without a forward must not reuse the prior batch.
|
||||
connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type]
|
||||
assert connector.kv_connector.start_load_kv.call_count == 2
|
||||
assert connector.kv_connector.start_load_kv.call_args.kwargs == {}
|
||||
|
||||
|
||||
def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
|
||||
events: list[str] = []
|
||||
|
||||
@@ -312,6 +312,10 @@ def test_select_common_block_size_uses_largest_shared_int():
|
||||
assert selected_size == 64
|
||||
|
||||
|
||||
def test_select_common_block_size_without_active_backends_uses_manager_size():
|
||||
assert select_common_block_size(256, []) == 256
|
||||
|
||||
|
||||
def test_select_common_block_size_accepts_rocm_sparse_block_size_16(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
|
||||
|
||||
@@ -237,6 +237,7 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner:
|
||||
runner.attn_groups = None
|
||||
runner.kv_cache_config = None
|
||||
runner.use_aux_hidden_state_outputs = False
|
||||
runner.kv_connector = model_runner_module.NO_OP_KV_CONNECTOR
|
||||
return runner
|
||||
|
||||
|
||||
|
||||
@@ -89,6 +89,7 @@ def _make_runner(
|
||||
return SimpleNamespace(
|
||||
num_speculative_steps=num_spec_steps,
|
||||
adaptive_verification=None,
|
||||
rejection_sampler=None,
|
||||
decode_query_len=num_spec_steps + 1,
|
||||
is_pooling_model=False,
|
||||
is_encoder_decoder=False,
|
||||
|
||||
@@ -5,8 +5,10 @@ from contextlib import AbstractContextManager
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as hisparse_binding
|
||||
import vllm.v1.worker.gpu.attn_utils as attn_utils
|
||||
import vllm.v1.worker.gpu_model_runner as gpu_model_runner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
@@ -26,7 +28,8 @@ class _AllocationScope(AbstractContextManager):
|
||||
self.active = False
|
||||
|
||||
|
||||
def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
|
||||
@pytest.mark.parametrize("hisparse", [False, True])
|
||||
def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> None:
|
||||
scope = _AllocationScope()
|
||||
kv_caches = {"layer": torch.empty(0)}
|
||||
|
||||
@@ -38,23 +41,48 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
|
||||
assert not scope.active
|
||||
|
||||
monkeypatch.setattr(attn_utils, "allocate_kv_cache", allocate)
|
||||
monkeypatch.setattr(attn_utils, "bind_kv_cache", bind)
|
||||
monkeypatch.setattr(attn_utils, "bind_kv_cache_to_layers", bind)
|
||||
monkeypatch.setattr(attn_utils, "get_shared_kv_cache_layers", lambda config: {})
|
||||
|
||||
hisparse_bindings = []
|
||||
if hisparse:
|
||||
host_pool = SimpleNamespace()
|
||||
|
||||
def bind_hisparse(**kwargs):
|
||||
hisparse_bindings.append(kwargs)
|
||||
assert scope.active
|
||||
assert kwargs["kv_caches"] is kv_caches
|
||||
assert kwargs["host_pool"] is host_pool
|
||||
return [
|
||||
SimpleNamespace(
|
||||
view=SimpleNamespace(cache=torch.empty(1, 1, 1), block_size=1),
|
||||
runtime=SimpleNamespace(),
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(hisparse_binding, "HiSparseHostPool", lambda: host_pool)
|
||||
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
|
||||
monkeypatch.setattr(hisparse_binding, "bind_hisparse_kv_caches", bind_hisparse)
|
||||
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(
|
||||
hisparse_config=object() if hisparse else None
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=1, max_num_batched_tokens=1),
|
||||
cache_config=SimpleNamespace(get_resolved_kv_cache_layout=lambda: None),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
|
||||
)
|
||||
result = attn_utils.init_kv_cache(
|
||||
[],
|
||||
{},
|
||||
SimpleNamespace(kv_cache_groups=[]),
|
||||
torch.device("cpu"),
|
||||
[],
|
||||
config,
|
||||
kv_cache_allocation_context=scope,
|
||||
block_tables=SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert len(hisparse_bindings) == int(hisparse)
|
||||
assert result is kv_caches
|
||||
assert not scope.active
|
||||
|
||||
|
||||
@@ -35,10 +35,15 @@ def test_mixed_warmup_skipped_for_single_seq(max_num_reqs):
|
||||
def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_warmup):
|
||||
"""Startup must warm fixed drafts before calibration and retain its manager."""
|
||||
manager = SimpleNamespace(cost_tables=None)
|
||||
runner = SimpleNamespace(adaptive_verification=manager)
|
||||
rejection_sampler = SimpleNamespace(enable_adaptive_verification=True)
|
||||
runner = SimpleNamespace(
|
||||
adaptive_verification=manager,
|
||||
rejection_sampler=rejection_sampler,
|
||||
)
|
||||
|
||||
def run_steps(model_runner, execute, sample):
|
||||
assert model_runner.adaptive_verification is None
|
||||
assert not model_runner.rejection_sampler.enable_adaptive_verification
|
||||
if fail_warmup:
|
||||
raise RuntimeError("warmup failed")
|
||||
|
||||
@@ -50,3 +55,4 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
||||
warmup.warmup_kernels(runner, _fail, _fail)
|
||||
assert runner.adaptive_verification is manager
|
||||
assert manager.cost_tables is None
|
||||
assert rejection_sampler.enable_adaptive_verification
|
||||
|
||||
@@ -1,13 +1,897 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.attention.backends.mla.index_group as index_group_module
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse import (
|
||||
worker as hisparse_worker_module,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
_flatten_row_mirrors,
|
||||
_select_written_row_mirrors,
|
||||
_SlotMappingStaging,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.v1.worker.utils import bind_kv_cache
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
|
||||
from vllm.v1.worker.utils import (
|
||||
bind_kv_cache,
|
||||
bind_kv_cache_to_layers,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
)
|
||||
|
||||
|
||||
def _make_hisparse_worker() -> HiSparseConnectorWorker:
|
||||
worker = object.__new__(HiSparseConnectorWorker)
|
||||
worker.refines_row_mirrors = False
|
||||
worker._slot_mapping_staging = None
|
||||
worker._row_mirror_num_rows = 0
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._pending_dma_descriptors = deque()
|
||||
worker._dma_free_descriptors = []
|
||||
worker.host_write_events = (MagicMock(), MagicMock())
|
||||
worker.host_write_event = worker.host_write_events[1]
|
||||
worker._next_host_write_event = 0
|
||||
worker.dma_stream = None
|
||||
worker.shared_host_region = None
|
||||
worker._completed_host_copy_dst_ids = []
|
||||
worker._slot_mapping_staging = None
|
||||
return worker
|
||||
|
||||
|
||||
def test_hisparse_row_mirrors_follow_runner_request_order():
|
||||
first = SparseKVRowMirror((1,), 10, 1)
|
||||
second = SparseKVRowMirror((2,), 20, 1)
|
||||
row_mirrors = {"first": (first,), "second": (second,)}
|
||||
|
||||
assert _flatten_row_mirrors(row_mirrors, ["second", "first"]) == (
|
||||
second,
|
||||
first,
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_gpu_slots_select_written_rows_from_async_envelope():
|
||||
"""Rejected speculative rows must never become durable host KV."""
|
||||
candidates = (
|
||||
SparseKVRowMirror((100, 500), 1000, 4),
|
||||
SparseKVRowMirror((20, 300), 2000, 4),
|
||||
)
|
||||
source_slots = np.array([102, 103, -1, 21, 22, 23], dtype=np.int64)
|
||||
|
||||
assert _select_written_row_mirrors(candidates, source_slots, 0) == (
|
||||
SparseKVRowMirror((102, 502), 1002, 2),
|
||||
SparseKVRowMirror((21, 301), 2001, 3),
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_written_rows_preserve_scheduler_page_boundaries():
|
||||
candidates = (
|
||||
SparseKVRowMirror((0,), 100, 4),
|
||||
SparseKVRowMirror((4,), 104, 4),
|
||||
)
|
||||
|
||||
assert _select_written_row_mirrors(
|
||||
candidates, np.array([2, 3, 4, 5], dtype=np.int64), 0
|
||||
) == (
|
||||
SparseKVRowMirror((2,), 102, 2),
|
||||
SparseKVRowMirror((4,), 104, 2),
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_written_rows_without_host_destinations_are_skipped():
|
||||
assert (
|
||||
_select_written_row_mirrors((), np.array([100, 101], dtype=np.int64), 0) == ()
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_appends_reference_slots_within_a_mirror_phase(monkeypatch):
|
||||
"""Separate context and query writes must share one mirror phase."""
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.refines_row_mirrors = True
|
||||
state = _SlotMappingStaging(
|
||||
MagicMock(), MagicMock(), torch.empty(4, dtype=torch.int64)
|
||||
)
|
||||
worker._slot_mapping_staging = state
|
||||
handle = SimpleNamespace(runtime=SimpleNamespace(resident_source_index=1))
|
||||
worker.cache_layer_names = ["layer"]
|
||||
worker.cache_handles = [handle]
|
||||
worker._layer_mirror_callbacks = (MagicMock(),)
|
||||
worker._row_mirror_num_rows = 1
|
||||
worker._submitted_mirror_layers = set()
|
||||
main_stream = MagicMock()
|
||||
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: main_stream)
|
||||
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
|
||||
|
||||
handle.slot_mapping = torch.tensor([7, 8], dtype=torch.int64)
|
||||
worker._stage_row_mirror_mapping(2)
|
||||
handle.slot_mapping = torch.tensor([9, 10], dtype=torch.int64)
|
||||
worker._stage_row_mirror_mapping(2)
|
||||
|
||||
torch.testing.assert_close(state.slots, torch.tensor([7, 8, 9, 10]))
|
||||
assert state.stream.wait_stream.call_args_list == [call(main_stream)] * 2
|
||||
assert state.event.record.call_args_list == [call(state.stream)] * 2
|
||||
assert state.num_tokens == 4
|
||||
assert state.source_index == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "expected_calls"),
|
||||
[(CUDAGraphMode.PIECEWISE, 1), (CUDAGraphMode.FULL, 0)],
|
||||
)
|
||||
def test_hisparse_submits_layer_mirror_at_replayed_attention_boundary(
|
||||
monkeypatch, mode, expected_calls
|
||||
):
|
||||
"""Piecewise replay must pipeline DMA; FULL replay uses the batch fallback."""
|
||||
handle = object.__new__(hisparse_runtime_module.HiSparseCacheHandle)
|
||||
handle.dummy_batch = False
|
||||
handle.decode_batch = False
|
||||
handle.submit_layer_mirror = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(cudagraph_runtime_mode=mode),
|
||||
)
|
||||
|
||||
handle.finish_kv_update()
|
||||
|
||||
assert handle.submit_layer_mirror.call_count == expected_calls
|
||||
|
||||
|
||||
def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
|
||||
backing = torch.full((10, 2, 3), -1, dtype=torch.float32)
|
||||
cache = backing[1:9]
|
||||
cache[2:4] = 7
|
||||
cache[6:8] = 11
|
||||
|
||||
copy_kv_cache_blocks_inplace(
|
||||
[cache],
|
||||
num_blocks=4,
|
||||
kv_cache_block_copies=[
|
||||
KVCacheBlockCopy(1, 0),
|
||||
KVCacheBlockCopy(3, 2),
|
||||
],
|
||||
)
|
||||
|
||||
torch.testing.assert_close(cache[0:2], torch.full_like(cache[0:2], 7))
|
||||
torch.testing.assert_close(cache[4:6], torch.full_like(cache[4:6], 11))
|
||||
assert (backing[0] == -1).all()
|
||||
assert (backing[9] == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
|
||||
@pytest.mark.parametrize("has_copies", [False, True])
|
||||
def test_hisparse_host_copy_waits_for_writes_on_writer_only(
|
||||
monkeypatch, shared, rank, has_copies
|
||||
):
|
||||
"""Host CoW must read completed writes; shared non-writers only barrier."""
|
||||
worker = _make_hisparse_worker()
|
||||
cache = torch.zeros((2, 1, 1))
|
||||
worker.host_caches = [cache]
|
||||
worker.host_num_blocks = 2
|
||||
worker.shared_host_region = object() if shared else None
|
||||
event = MagicMock()
|
||||
event.synchronize.side_effect = lambda: cache[0].fill_(7)
|
||||
barrier = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "get_tensor_model_parallel_rank", lambda: rank
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "get_tp_group", lambda: SimpleNamespace(barrier=barrier)
|
||||
)
|
||||
copies = [KVCacheBlockCopy(0, 1)] if has_copies else []
|
||||
|
||||
worker._copy_host_blocks(copies, event)
|
||||
|
||||
writes = has_copies and (not shared or rank == 0)
|
||||
assert cache[1].item() == (7 if writes else 0)
|
||||
assert event.synchronize.call_count == int(writes)
|
||||
assert barrier.call_count == int(shared and has_copies)
|
||||
assert worker._completed_host_copy_dst_ids == ([1] if has_copies else [])
|
||||
|
||||
|
||||
def test_hisparse_worker_updates_request_state_mapping_in_place(monkeypatch):
|
||||
worker = _make_hisparse_worker()
|
||||
worker.request_state_indices = torch.arange(4, dtype=torch.int32)
|
||||
worker._pending_invalid_block_ids = [5]
|
||||
invalidations = []
|
||||
worker.invalidate_blocks = lambda blocks, states: invalidations.append(
|
||||
(blocks.copy(), states.clone())
|
||||
)
|
||||
original_ptr = worker.request_state_indices.data_ptr()
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
|
||||
worker.set_request_state_indices(torch.tensor([3, 1], dtype=torch.int32))
|
||||
|
||||
assert worker.request_state_indices.data_ptr() == original_ptr
|
||||
assert worker.request_state_indices.tolist() == [3, 1, -1, -1]
|
||||
assert len(invalidations) == 1
|
||||
assert invalidations[0][0] == [5]
|
||||
torch.testing.assert_close(
|
||||
invalidations[0][1], torch.tensor([3, 1], dtype=torch.int32)
|
||||
)
|
||||
assert worker._pending_invalid_block_ids == []
|
||||
|
||||
|
||||
def test_hisparse_pre_forward_transfer_builds_page_descriptors():
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.kernel_block_size = 2
|
||||
source = torch.empty((3, 2, 4), dtype=torch.uint8)
|
||||
destination = torch.empty((6, 4), dtype=torch.uint8)
|
||||
worker.resident_caches = (source,)
|
||||
worker.host_caches = (destination,)
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(runtime=SimpleNamespace(resident_source_index=0))
|
||||
]
|
||||
worker._dma_free_descriptors = []
|
||||
worker._submit_dma_descriptors = MagicMock()
|
||||
|
||||
worker._enqueue_transfers([SparseKVPageTransfer(7, 2, (1,), after_forward=False)])
|
||||
|
||||
descriptors, count = worker._submit_dma_descriptors.call_args.args
|
||||
assert count == 1
|
||||
assert descriptors.src[:count].tolist() == [
|
||||
source.data_ptr() + source.stride(0) * source.element_size()
|
||||
]
|
||||
assert descriptors.dst[:count].tolist() == [destination.data_ptr() + 4 * 4]
|
||||
assert descriptors.sizes[:count].tolist() == [2 * 4]
|
||||
assert worker._submit_dma_descriptors.call_args.kwargs["transfer_ids"] == (7,)
|
||||
|
||||
|
||||
def test_hisparse_eager_mirror_records_transfer_without_page_copy():
|
||||
worker = _make_hisparse_worker()
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(runtime=SimpleNamespace(eager_host_mirror=True))
|
||||
]
|
||||
worker._record_transfer_completion = MagicMock()
|
||||
worker._enqueue_transfers = MagicMock()
|
||||
transfers = [SparseKVPageTransfer(7, 2, (1,), after_forward=False)]
|
||||
|
||||
worker._submit_transfers(transfers)
|
||||
|
||||
worker._record_transfer_completion.assert_called_once_with(transfers)
|
||||
worker._enqueue_transfers.assert_not_called()
|
||||
|
||||
|
||||
def test_hisparse_lazy_mirror_copies_transfer_pages():
|
||||
worker = _make_hisparse_worker()
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(runtime=SimpleNamespace(eager_host_mirror=False))
|
||||
]
|
||||
worker._record_transfer_completion = MagicMock()
|
||||
worker._enqueue_transfers = MagicMock()
|
||||
transfers = [SparseKVPageTransfer(7, 2, (1,), after_forward=False)]
|
||||
|
||||
worker._submit_transfers(transfers)
|
||||
|
||||
worker._enqueue_transfers.assert_called_once_with(transfers)
|
||||
worker._record_transfer_completion.assert_not_called()
|
||||
|
||||
|
||||
def test_hisparse_dma_row_mirror_builds_descriptors(monkeypatch):
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.kernel_block_size = 2
|
||||
source = torch.empty((3, 2, 4), dtype=torch.uint8)
|
||||
destination = torch.empty((6, 4), dtype=torch.uint8)
|
||||
worker.resident_caches = (source,)
|
||||
worker.host_caches = (destination,)
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(
|
||||
runtime=SimpleNamespace(resident_source_index=0), decode_batch=True
|
||||
)
|
||||
]
|
||||
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
|
||||
worker.dma_stream = MagicMock()
|
||||
worker.host_write_event = MagicMock()
|
||||
worker._dma_free_descriptors = []
|
||||
worker._pending_dma_descriptors = deque()
|
||||
worker._pending_transfer_events = deque()
|
||||
worker._enqueued_transfer_ids = []
|
||||
worker._dma_submitted = False
|
||||
current_stream = MagicMock()
|
||||
event = MagicMock()
|
||||
swap_blocks_batch = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "current_stream", lambda: current_stream
|
||||
)
|
||||
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
|
||||
monkeypatch.setattr(torch, "Event", lambda: event)
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module.ops, "swap_blocks_batch", swap_blocks_batch
|
||||
)
|
||||
|
||||
worker._set_row_mirrors((SparseKVRowMirror((3,), 4, 1),))
|
||||
worker._enqueue_row_dma(range(1))
|
||||
|
||||
src_ptrs, dst_ptrs, sizes = swap_blocks_batch.call_args.args
|
||||
assert src_ptrs.tolist() == [source.data_ptr() + 3 * 4]
|
||||
assert dst_ptrs.tolist() == [destination.data_ptr() + 4 * 4]
|
||||
assert sizes.tolist() == [4]
|
||||
worker.dma_stream.wait_stream.assert_called_once_with(current_stream)
|
||||
worker.host_write_event.record.assert_called_once_with(worker.dma_stream)
|
||||
event.record.assert_called_once_with(worker.dma_stream)
|
||||
assert worker._dma_submitted
|
||||
|
||||
|
||||
def test_hisparse_row_dma_uses_resident_spans():
|
||||
worker = _make_hisparse_worker()
|
||||
source = torch.empty((2, 2, 4), dtype=torch.uint8)
|
||||
destination = torch.empty((12, 4), dtype=torch.uint8)
|
||||
worker.is_host_writer = True
|
||||
worker.kernel_block_size = 2
|
||||
worker.resident_caches = (source,)
|
||||
worker.host_caches = (destination,)
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(
|
||||
runtime=SimpleNamespace(resident_source_index=0), decode_batch=True
|
||||
)
|
||||
]
|
||||
worker._set_row_mirrors(
|
||||
(
|
||||
SparseKVRowMirror((0,), 4, 2),
|
||||
SparseKVRowMirror((2,), 9, 1),
|
||||
)
|
||||
)
|
||||
worker._dma_free_descriptors = []
|
||||
worker._submit_dma_descriptors = MagicMock()
|
||||
|
||||
worker._enqueue_row_dma(range(1))
|
||||
|
||||
descriptors, count = worker._submit_dma_descriptors.call_args.args
|
||||
assert count == 2
|
||||
assert descriptors.src[:count].tolist() == [
|
||||
source.data_ptr(),
|
||||
source.data_ptr() + 2 * 4,
|
||||
]
|
||||
assert descriptors.dst[:count].tolist() == [
|
||||
destination.data_ptr() + 4 * 4,
|
||||
destination.data_ptr() + 9 * 4,
|
||||
]
|
||||
assert descriptors.sizes[:count].tolist() == [2 * 4, 4]
|
||||
|
||||
|
||||
def test_hisparse_finish_forward_mirrors_all_layers_once(monkeypatch):
|
||||
dst_slots = torch.tensor([7, 8, 9], dtype=torch.int64)
|
||||
req_ids = torch.tensor([0, 1, 2], dtype=torch.int32)
|
||||
leader = SimpleNamespace(
|
||||
eager_host_mirror=True,
|
||||
is_group_leader=True,
|
||||
invalidate_written_slots=MagicMock(),
|
||||
)
|
||||
follower = SimpleNamespace(
|
||||
eager_host_mirror=True,
|
||||
is_group_leader=False,
|
||||
invalidate_written_slots=MagicMock(),
|
||||
)
|
||||
handles = [
|
||||
SimpleNamespace(
|
||||
runtime=runtime,
|
||||
decode_batch=True,
|
||||
host_mirror_required=True,
|
||||
num_actual_tokens=3,
|
||||
num_decode_tokens=2,
|
||||
req_id_per_token=req_ids,
|
||||
mirror_slot_mapping=dst_slots,
|
||||
)
|
||||
for runtime in (leader, follower)
|
||||
]
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.cache_handles = handles
|
||||
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
|
||||
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 3),))
|
||||
worker._dma_submitted = False
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._post_forward_transfers = []
|
||||
worker._enqueue_row_dma = MagicMock()
|
||||
worker.host_write_event = MagicMock()
|
||||
worker._forward_ready_event = MagicMock()
|
||||
current_stream = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "current_stream", lambda: current_stream
|
||||
)
|
||||
|
||||
worker.finish_forward()
|
||||
|
||||
worker._forward_ready_event.record.assert_called_once_with()
|
||||
worker._enqueue_row_dma.assert_called_once_with(
|
||||
(0, 1), ready_event=worker._forward_ready_event
|
||||
)
|
||||
leader.invalidate_written_slots.assert_called_once()
|
||||
torch.testing.assert_close(
|
||||
leader.invalidate_written_slots.call_args.args[0], dst_slots
|
||||
)
|
||||
follower.invalidate_written_slots.assert_not_called()
|
||||
worker.host_write_event.record.assert_called_once_with(current_stream)
|
||||
|
||||
|
||||
def test_hisparse_finish_forward_does_not_repeat_per_layer_mirrors():
|
||||
slots = torch.tensor([7, 8], dtype=torch.int64)
|
||||
runtime = SimpleNamespace(
|
||||
eager_host_mirror=False,
|
||||
is_group_leader=False,
|
||||
invalidate_written_slots=MagicMock(),
|
||||
)
|
||||
handles = [
|
||||
SimpleNamespace(
|
||||
runtime=runtime,
|
||||
decode_batch=False,
|
||||
host_mirror_required=True,
|
||||
num_actual_tokens=2,
|
||||
num_decode_tokens=0,
|
||||
req_id_per_token=torch.empty(0, dtype=torch.int32),
|
||||
mirror_slot_mapping=slots,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.cache_handles = handles
|
||||
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 2),))
|
||||
worker._per_layer_mirrored = {0, 1}
|
||||
worker._submitted_mirror_layers = {0, 1}
|
||||
worker._enqueue_row_dma = MagicMock()
|
||||
|
||||
worker._enqueue_host_mirror()
|
||||
|
||||
worker._enqueue_row_dma.assert_not_called()
|
||||
|
||||
|
||||
def test_hisparse_finish_forward_submits_lazy_post_forward_transfer(monkeypatch):
|
||||
runtime = SimpleNamespace(eager_host_mirror=False)
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.cache_handles = [SimpleNamespace(runtime=runtime, num_actual_tokens=0)]
|
||||
transfer = SparseKVPageTransfer(7, 2, (1,), after_forward=True)
|
||||
worker._post_forward_transfers = [transfer]
|
||||
worker._forward_ready_event = MagicMock()
|
||||
worker._enqueue_host_mirror = MagicMock()
|
||||
worker._submit_transfers = MagicMock()
|
||||
worker._dma_submitted = False
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._finish_mirror_phase = MagicMock()
|
||||
worker._release_completed_dma_descriptors = MagicMock()
|
||||
stream = MagicMock()
|
||||
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
|
||||
|
||||
worker.finish_forward()
|
||||
|
||||
worker._finish_mirror_phase.assert_called_once_with(worker._forward_ready_event)
|
||||
worker._submit_transfers.assert_called_once_with([transfer])
|
||||
worker.host_write_event.record.assert_called_once_with(stream)
|
||||
|
||||
|
||||
def test_hisparse_prefill_mirrors_source_groups_and_flushes_partial_group():
|
||||
slots = torch.tensor([7, 8], dtype=torch.int64)
|
||||
source_indices = [0, 0, 1, 1, 1, 1, 2]
|
||||
handles = [
|
||||
SimpleNamespace(
|
||||
runtime=SimpleNamespace(
|
||||
eager_host_mirror=False,
|
||||
is_group_leader=False,
|
||||
resident_source_index=source_index,
|
||||
),
|
||||
decode_batch=False,
|
||||
host_mirror_required=layer_index < 5,
|
||||
num_actual_tokens=2 if layer_index < 5 else 0,
|
||||
num_decode_tokens=0,
|
||||
req_id_per_token=torch.empty(0, dtype=torch.int32),
|
||||
mirror_slot_mapping=slots,
|
||||
)
|
||||
for layer_index, source_index in enumerate(source_indices)
|
||||
]
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.cache_handles = handles
|
||||
worker._set_row_mirrors((SparseKVRowMirror((0, 0, 0), 7, 2),))
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._layer_ready_events = tuple(MagicMock() for _ in handles)
|
||||
worker._enqueue_row_dma = MagicMock()
|
||||
|
||||
for layer_index in range(5):
|
||||
worker._enqueue_layer_mirror(layer_index)
|
||||
|
||||
worker._enqueue_row_dma.assert_called_once_with(
|
||||
(0, 1), ready_event=worker._layer_ready_events[1]
|
||||
)
|
||||
worker._enqueue_host_mirror(ready_event=worker._layer_ready_events[-1])
|
||||
assert worker._enqueue_row_dma.call_args_list[1].args == ((2, 3, 4),)
|
||||
assert worker._enqueue_row_dma.call_args_list[1].kwargs == {
|
||||
"ready_event": worker._layer_ready_events[-1]
|
||||
}
|
||||
|
||||
|
||||
def test_hisparse_finish_forward_rejects_partial_per_layer_mirror():
|
||||
slots = torch.tensor([7, 8], dtype=torch.int64)
|
||||
runtime = SimpleNamespace(
|
||||
eager_host_mirror=False,
|
||||
is_group_leader=False,
|
||||
invalidate_written_slots=MagicMock(),
|
||||
)
|
||||
handles = [
|
||||
SimpleNamespace(
|
||||
runtime=runtime,
|
||||
decode_batch=False,
|
||||
host_mirror_required=True,
|
||||
num_actual_tokens=2,
|
||||
num_decode_tokens=0,
|
||||
req_id_per_token=torch.empty(0, dtype=torch.int32),
|
||||
mirror_slot_mapping=slots,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.cache_handles = handles
|
||||
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 2),))
|
||||
worker._per_layer_mirrored = {0}
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._enqueue_row_dma = MagicMock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="did not mirror every active layer"):
|
||||
worker._enqueue_host_mirror()
|
||||
|
||||
|
||||
def test_hisparse_finish_forward_orders_next_forward_after_dma(monkeypatch):
|
||||
current_stream = MagicMock()
|
||||
worker = _make_hisparse_worker()
|
||||
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
|
||||
worker.is_host_writer = True
|
||||
worker._dma_submitted = True
|
||||
worker.host_write_event = MagicMock()
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(
|
||||
runtime=SimpleNamespace(eager_host_mirror=True), num_actual_tokens=0
|
||||
)
|
||||
]
|
||||
worker._post_forward_transfers = []
|
||||
worker._pending_dma_descriptors = deque()
|
||||
worker._dma_free_descriptors = []
|
||||
worker._forward_ready_event = MagicMock()
|
||||
worker._finish_mirror_phase = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "current_stream", lambda: current_stream
|
||||
)
|
||||
|
||||
worker.finish_forward()
|
||||
|
||||
current_stream.wait_event.assert_called_once_with(worker.host_write_event)
|
||||
assert not worker._dma_submitted
|
||||
|
||||
|
||||
def test_hisparse_finish_forward_mirrors_standalone_mtp_cache(monkeypatch):
|
||||
dst_slots = torch.tensor([7, 8], dtype=torch.int64)
|
||||
runtime = SimpleNamespace(
|
||||
eager_host_mirror=True,
|
||||
is_group_leader=False,
|
||||
invalidate_written_slots=MagicMock(),
|
||||
)
|
||||
inactive = SimpleNamespace(
|
||||
runtime=runtime,
|
||||
decode_batch=False,
|
||||
host_mirror_required=False,
|
||||
num_actual_tokens=0,
|
||||
num_decode_tokens=0,
|
||||
req_id_per_token=None,
|
||||
mirror_slot_mapping=dst_slots,
|
||||
)
|
||||
mtp = SimpleNamespace(
|
||||
runtime=runtime,
|
||||
decode_batch=True,
|
||||
host_mirror_required=True,
|
||||
num_actual_tokens=2,
|
||||
num_decode_tokens=2,
|
||||
req_id_per_token=torch.tensor([0, 1], dtype=torch.int32),
|
||||
mirror_slot_mapping=dst_slots,
|
||||
)
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.cache_handles = [inactive, mtp]
|
||||
worker._set_row_mirrors((SparseKVRowMirror((0,), 7, 2),))
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._enqueue_row_dma = MagicMock()
|
||||
|
||||
worker._enqueue_host_mirror()
|
||||
|
||||
worker._enqueue_row_dma.assert_called_once_with((1,), ready_event=None)
|
||||
|
||||
|
||||
def test_hisparse_shared_host_reader_skips_mirror(monkeypatch):
|
||||
"""A non-writer TP rank must not mirror rows into the shared host pool."""
|
||||
dst_slots = torch.tensor([7, 8], dtype=torch.int64)
|
||||
leader = SimpleNamespace(
|
||||
eager_host_mirror=True,
|
||||
is_group_leader=True,
|
||||
invalidate_written_slots=MagicMock(),
|
||||
)
|
||||
handle = SimpleNamespace(
|
||||
runtime=leader,
|
||||
decode_batch=True,
|
||||
host_mirror_required=True,
|
||||
num_actual_tokens=2,
|
||||
num_decode_tokens=1,
|
||||
req_id_per_token=torch.tensor([0], dtype=torch.int32),
|
||||
mirror_slot_mapping=dst_slots,
|
||||
)
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = False
|
||||
worker.cache_handles = [handle]
|
||||
worker._set_row_mirrors((SparseKVRowMirror((0,), 7, 2),))
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._enqueue_row_dma = MagicMock()
|
||||
|
||||
worker._enqueue_host_mirror()
|
||||
|
||||
worker._enqueue_row_dma.assert_not_called()
|
||||
leader.invalidate_written_slots.assert_called_once()
|
||||
|
||||
|
||||
def test_hisparse_shared_host_reader_skips_transfer_completion():
|
||||
"""A non-writer TP rank must not acknowledge host writes."""
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = False
|
||||
worker.kernel_block_size = 1
|
||||
worker._enqueued_transfer_ids = []
|
||||
worker._pending_transfer_events = []
|
||||
|
||||
worker._record_transfer_completion([SparseKVPageTransfer(1, 2, (3,), True)])
|
||||
|
||||
assert worker._enqueued_transfer_ids == []
|
||||
assert worker._pending_transfer_events == []
|
||||
|
||||
|
||||
def test_hisparse_writer_records_transfer_completion_after_dma(monkeypatch):
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.dma_stream = MagicMock()
|
||||
worker._enqueued_transfer_ids = []
|
||||
worker._pending_transfer_events = deque()
|
||||
event = MagicMock()
|
||||
monkeypatch.setattr(torch, "Event", lambda: event)
|
||||
|
||||
worker._record_transfer_completion(
|
||||
[
|
||||
SparseKVPageTransfer(3, 2, (1,), False),
|
||||
SparseKVPageTransfer(7, 4, (5,), True),
|
||||
]
|
||||
)
|
||||
|
||||
event.record.assert_called_once_with(worker.dma_stream)
|
||||
assert worker._enqueued_transfer_ids == [3, 7]
|
||||
assert worker._pending_transfer_events == deque([(event, (3, 7))])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_host_writer", [False, True])
|
||||
def test_hisparse_step_waits_for_previous_host_write(monkeypatch, is_host_writer):
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = is_host_writer
|
||||
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:1"))
|
||||
host_write_events = (MagicMock(), MagicMock())
|
||||
worker.host_write_events = host_write_events
|
||||
worker.host_write_event = host_write_events[1]
|
||||
worker._next_host_write_event = 0
|
||||
worker.host_caches = ()
|
||||
worker.host_num_blocks = 1
|
||||
worker._post_forward_transfers = []
|
||||
worker._pending_invalid_block_ids = []
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(
|
||||
runtime=SimpleNamespace(eager_host_mirror=True),
|
||||
decode_batch=False,
|
||||
host_mirror_required=False,
|
||||
num_actual_tokens=0,
|
||||
num_decode_tokens=0,
|
||||
req_id_per_token=None,
|
||||
)
|
||||
]
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._layer_mirror_callbacks = ()
|
||||
stream = MagicMock()
|
||||
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
|
||||
|
||||
worker.start_step(
|
||||
SimpleNamespace(
|
||||
host_block_copies=[],
|
||||
command=None,
|
||||
source_block_ids=[],
|
||||
row_mirrors={},
|
||||
all_context_pages_resident=True,
|
||||
row_mirrors_from_resident=False,
|
||||
),
|
||||
None,
|
||||
)
|
||||
worker.start_step(
|
||||
SimpleNamespace(
|
||||
host_block_copies=[],
|
||||
command=None,
|
||||
source_block_ids=[],
|
||||
row_mirrors={},
|
||||
all_context_pages_resident=True,
|
||||
row_mirrors_from_resident=False,
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
assert stream.wait_event.call_args_list == [
|
||||
call(host_write_events[1]),
|
||||
call(host_write_events[0]),
|
||||
]
|
||||
assert worker.host_write_event is host_write_events[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank, expected_copies", [(0, 1), (1, 0)])
|
||||
def test_hisparse_shared_host_block_copy_has_one_writer(
|
||||
monkeypatch, tp_rank, expected_copies
|
||||
):
|
||||
worker = _make_hisparse_worker()
|
||||
worker.shared_host_region = object()
|
||||
worker.host_caches = (torch.empty(4, 1),)
|
||||
worker.host_num_blocks = 4
|
||||
previous_event = MagicMock()
|
||||
copies = (KVCacheBlockCopy(0, 1),)
|
||||
copy_blocks = MagicMock()
|
||||
tp_group = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "get_tensor_model_parallel_rank", lambda: tp_rank
|
||||
)
|
||||
monkeypatch.setattr(hisparse_worker_module, "get_tp_group", lambda: tp_group)
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "copy_kv_cache_blocks_inplace", copy_blocks
|
||||
)
|
||||
|
||||
worker._copy_host_blocks(copies, previous_event)
|
||||
|
||||
assert copy_blocks.call_count == expected_copies
|
||||
tp_group.barrier.assert_called_once_with()
|
||||
assert worker.take_completed_host_copies() == [1]
|
||||
assert worker.take_completed_host_copies() == []
|
||||
|
||||
|
||||
def test_hisparse_empty_step_does_not_replay_stale_host_mirror(monkeypatch):
|
||||
handle = SimpleNamespace(
|
||||
runtime=SimpleNamespace(eager_host_mirror=True),
|
||||
decode_batch=True,
|
||||
num_actual_tokens=2,
|
||||
num_decode_tokens=2,
|
||||
req_id_per_token=torch.tensor([0, 1]),
|
||||
mirror_slot_mapping=torch.tensor([4, 5]),
|
||||
)
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = True
|
||||
worker.hot_backing = SimpleNamespace(device=torch.device("cpu"))
|
||||
worker.host_write_event = MagicMock()
|
||||
worker._forward_ready_event = MagicMock()
|
||||
worker.host_caches = ()
|
||||
worker.host_num_blocks = 1
|
||||
worker.cache_handles = [handle]
|
||||
worker._per_layer_mirrored = set()
|
||||
worker._submitted_mirror_layers = set()
|
||||
worker._layer_mirror_callbacks = (MagicMock(),)
|
||||
worker._post_forward_transfers = []
|
||||
worker._pending_invalid_block_ids = []
|
||||
worker._enqueue_host_mirror = MagicMock(wraps=worker._enqueue_host_mirror)
|
||||
stream = MagicMock()
|
||||
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
|
||||
|
||||
worker.start_step(
|
||||
SimpleNamespace(
|
||||
host_block_copies=[],
|
||||
command=None,
|
||||
source_block_ids=[],
|
||||
row_mirrors={},
|
||||
all_context_pages_resident=True,
|
||||
row_mirrors_from_resident=False,
|
||||
),
|
||||
None,
|
||||
)
|
||||
worker.finish_forward()
|
||||
|
||||
worker._enqueue_host_mirror.assert_called_once_with(worker._forward_ready_event)
|
||||
assert handle.num_actual_tokens == 0
|
||||
torch.testing.assert_close(handle.mirror_slot_mapping, torch.tensor([4, 5]))
|
||||
|
||||
|
||||
def test_hisparse_cache_handles_join_index_groups_during_construction(monkeypatch):
|
||||
"""Followers must not allocate duplicate runtime state before profiling."""
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=2,
|
||||
async_scheduling=False,
|
||||
),
|
||||
speculative_config=None,
|
||||
kv_transfer_config=None,
|
||||
)
|
||||
resolved = hisparse_runtime_module.ResolvedHiSparseConfig(
|
||||
top_k=4,
|
||||
device_buffer_size=8,
|
||||
)
|
||||
monkeypatch.setattr(hisparse_runtime_module, "_has_hisparse_ops", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.ResolvedHiSparseConfig,
|
||||
"from_vllm_config",
|
||||
classmethod(lambda cls, vllm_config, model_top_k: resolved),
|
||||
)
|
||||
shared_states: list[object] = []
|
||||
streams: list[object] = []
|
||||
|
||||
def create_shared_state(_device, _max_rows, _top_k):
|
||||
shared_states.append(object())
|
||||
return shared_states[-1]
|
||||
|
||||
def create_stream(_device):
|
||||
streams.append(object())
|
||||
return streams[-1]
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "_create_shared_topk_state", create_shared_state
|
||||
)
|
||||
monkeypatch.setattr(hisparse_runtime_module, "_create_copy_stream", create_stream)
|
||||
monkeypatch.setattr(index_group_module, "_create_side_stream", lambda _: object())
|
||||
monkeypatch.setattr(index_group_module, "_create_event", lambda: object())
|
||||
index_group_builder = index_group_module.SparseMLAIndexGroupBuilder(
|
||||
torch.empty((2, 4), dtype=torch.int32)
|
||||
)
|
||||
|
||||
def make_cache_handle(is_leader: bool):
|
||||
index_group, _ = index_group_builder.register_layer(is_leader)
|
||||
cache_handle = hisparse_runtime_module.create_hisparse_cache_handle(
|
||||
config,
|
||||
model_top_k=4,
|
||||
is_index_group_leader=is_leader,
|
||||
row_width=8,
|
||||
kv_dtype=torch.float32,
|
||||
index_group=index_group,
|
||||
device="cpu",
|
||||
)
|
||||
assert cache_handle is not None
|
||||
return cache_handle
|
||||
|
||||
first_leader = make_cache_handle(True)
|
||||
first_index_group = index_group_builder.current_group
|
||||
first_follower = make_cache_handle(False)
|
||||
second_leader = make_cache_handle(True)
|
||||
second_index_group = index_group_builder.current_group
|
||||
second_follower = make_cache_handle(False)
|
||||
|
||||
assert first_index_group is not None
|
||||
assert second_index_group is not None
|
||||
assert first_follower.runtime.index_group is first_leader.runtime.index_group
|
||||
assert second_follower.runtime.index_group is second_leader.runtime.index_group
|
||||
assert first_leader.runtime.index_group is not second_leader.runtime.index_group
|
||||
assert first_leader.runtime.index_group.copy_stream is first_index_group.side_stream
|
||||
assert (
|
||||
first_leader.runtime.index_group.logical_topk_ready
|
||||
is first_index_group.logical_topk_ready
|
||||
)
|
||||
assert (
|
||||
second_leader.runtime.index_group.copy_stream is second_index_group.side_stream
|
||||
)
|
||||
assert (
|
||||
second_leader.runtime.index_group.logical_topk_ready
|
||||
is second_index_group.logical_topk_ready
|
||||
)
|
||||
assert len(shared_states) == 2
|
||||
assert streams == []
|
||||
|
||||
|
||||
class _TestReplaySSMMixer(MambaMixer2):
|
||||
@@ -30,7 +914,8 @@ def _packed_replayssm_cache(num_blocks: int) -> torch.Tensor:
|
||||
return torch.full((num_blocks, 1, 1, 80), 0, dtype=torch.int8)
|
||||
|
||||
|
||||
def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group():
|
||||
@pytest.mark.parametrize("layers_only", [False, True])
|
||||
def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group(layers_only):
|
||||
mixers = [_TestReplaySSMMixer() for _ in range(3)]
|
||||
layer_names = [f"layers.{i}.mixer" for i in range(3)]
|
||||
ctx = dict(zip(layer_names, mixers))
|
||||
@@ -45,7 +930,10 @@ def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group():
|
||||
SimpleNamespace(layer_names=[layer_names[1]]),
|
||||
]
|
||||
|
||||
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
|
||||
if layers_only:
|
||||
bind_kv_cache_to_layers(kv_cache, ctx, kv_cache_groups=kv_cache_groups)
|
||||
else:
|
||||
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
|
||||
|
||||
assert (
|
||||
mixers[0]._replayssm_ring_start.data_ptr()
|
||||
|
||||
Reference in New Issue
Block a user