Merge branch 'main' into patches
This commit is contained in:
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
cu_num_tokens,
|
||||
cu_num_tokens[1:] - cu_num_tokens[:-1],
|
||||
KERNEL_BLOCK_SIZE=kernel_block_size,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ Known Issues:
|
||||
|
||||
import sys
|
||||
from types import MethodType, SimpleNamespace
|
||||
from typing import Literal
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -47,6 +48,9 @@ from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
MLAPrefillSelectorConfig,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
|
||||
is_cute_dsl_mla_prefill_supported,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -418,6 +422,11 @@ for backend in BACKENDS_TO_TEST:
|
||||
if query_len_support != QueryLenSupport.SINGLE_ONLY:
|
||||
SPEC_DECODE_BACKENDS.append(backend)
|
||||
|
||||
# Backends whose decode pipeline replays device-trimmed verify batches.
|
||||
ADAPTIVE_VARLEN_BACKENDS = [
|
||||
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
|
||||
]
|
||||
|
||||
BACKEND_BLOCK_SIZES = {}
|
||||
for backend in BACKENDS_TO_TEST:
|
||||
supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
|
||||
@@ -470,6 +479,10 @@ BATCH_SPECS = {
|
||||
"spec_decode_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
|
||||
),
|
||||
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
|
||||
"spec_decode_ragged_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
|
||||
),
|
||||
"spec_decode_medium": BatchSpec(
|
||||
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
|
||||
),
|
||||
@@ -1474,6 +1487,7 @@ def run_attention_backend(
|
||||
|
||||
# Force the prefill backend selection (None means auto-select).
|
||||
vllm_config.attention_config.mla_prefill_backend = prefill_backend
|
||||
mla_attention_module.backend_supports_prefill_query_quantization.cache_clear()
|
||||
|
||||
# Set the current vllm config so that get_current_vllm_config() works
|
||||
# in the backend implementations
|
||||
@@ -1581,6 +1595,42 @@ def run_attention_backend(
|
||||
return output
|
||||
|
||||
|
||||
def _with_even_cpu_query_split(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> CommonAttentionMetadata:
|
||||
"""Host query lengths as adaptive verification leaves them.
|
||||
|
||||
The device keeps the trimmed per-request lengths; the host sees the same
|
||||
total spread evenly, the first requests taking the remainder
|
||||
(AdaptiveVerificationManager.compact_batch).
|
||||
"""
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
|
||||
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
|
||||
cpu_lens[: total % num_reqs] += 1
|
||||
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
|
||||
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
|
||||
return common_attn_metadata.replace(
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=int(
|
||||
(
|
||||
common_attn_metadata.query_start_loc[1:]
|
||||
- common_attn_metadata.query_start_loc[:-1]
|
||||
)
|
||||
.max()
|
||||
.item()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _fp8_cache_roundtrip(tensor: torch.Tensor, scale: float) -> torch.Tensor:
|
||||
"""Reproduce cache storage rounding for the SDPA reference."""
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
limits = torch.finfo(fp8_dtype)
|
||||
quantized = (tensor.float() / scale).clamp(limits.min, limits.max).to(fp8_dtype)
|
||||
return (quantized.float() * scale).to(tensor.dtype)
|
||||
|
||||
|
||||
def _run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -1595,6 +1645,9 @@ def _run_backend_correctness(
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
adaptive_verification: bool = False,
|
||||
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"] = "trtllm-gen",
|
||||
use_prefill_query_quantization: bool = False,
|
||||
):
|
||||
"""Test that all backends produce similar outputs to a reference implementation
|
||||
using torch.nn.functional.scaled_dot_product_attention.
|
||||
@@ -1626,6 +1679,10 @@ def _run_backend_correctness(
|
||||
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
|
||||
# CUTLASS_MLA does not support non-1 Q/K scales
|
||||
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
|
||||
if adaptive_verification:
|
||||
backends_to_test = [
|
||||
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
|
||||
]
|
||||
if not backends_to_test:
|
||||
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
|
||||
|
||||
@@ -1670,18 +1727,25 @@ def _run_backend_correctness(
|
||||
hf_config_override["num_attention_heads"]
|
||||
)
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.attention_config.trtllm_ragged_backend = trtllm_ragged_backend
|
||||
vllm_config.attention_config.use_prefill_query_quantization = (
|
||||
use_prefill_query_quantization
|
||||
)
|
||||
|
||||
# For spec decode tests, add a speculative_config to set the reorder_batch_threshold
|
||||
if is_spec_decode_test:
|
||||
from vllm.config import SpeculativeConfig
|
||||
|
||||
# Get the query length from the batch spec (they should all be uniform)
|
||||
query_len = batch_spec.query_lens[0]
|
||||
# Get the query length from the batch spec (they should all be uniform,
|
||||
# except under adaptive verification, where the longest is k+1)
|
||||
query_len = max(batch_spec.query_lens)
|
||||
# Set num_speculative_tokens to query_len - 1
|
||||
# (since threshold is 1 + num_spec_tokens)
|
||||
# Use ngram method which doesn't require a draft model
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=query_len - 1
|
||||
method="ngram",
|
||||
num_speculative_tokens=query_len - 1,
|
||||
enable_adaptive_verification=adaptive_verification,
|
||||
)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
@@ -1756,6 +1820,19 @@ def _run_backend_correctness(
|
||||
# K_PE (rope component): [s_len, 1, qk_rope_head_dim]
|
||||
k_pe_full = torch.randn(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
|
||||
|
||||
kv_c_cached, k_pe_cached = kv_c_full, k_pe_full
|
||||
kv_c_prefill, k_pe_prefill = kv_c_full, k_pe_full
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
kv_c_cached = _fp8_cache_roundtrip(kv_c_full, k_scale)
|
||||
k_pe_cached = _fp8_cache_roundtrip(k_pe_full, k_scale)
|
||||
# Prefill reads cached context and unquantized new tokens.
|
||||
kv_c_prefill = torch.cat(
|
||||
[kv_c_cached[:context_len], kv_c_full[context_len:]]
|
||||
)
|
||||
k_pe_prefill = torch.cat(
|
||||
[k_pe_cached[:context_len], k_pe_full[context_len:]]
|
||||
)
|
||||
|
||||
# Determine if this sequence uses the decode pipeline or prefill
|
||||
# pipeline for each backend
|
||||
# NOTE: For spec decode tests with uniform query_len > 1, backends that
|
||||
@@ -1805,10 +1882,10 @@ def _run_backend_correctness(
|
||||
q_mqa = torch.cat([ql_nope, q_pe], dim=-1)
|
||||
# K: [s_len, kv_lora_rank + qk_rope_head_dim]
|
||||
# (broadcasted to all heads)
|
||||
k_mqa = torch.cat([kv_c_full, k_pe_full.squeeze(1)], dim=-1)
|
||||
k_mqa = torch.cat([kv_c_cached, k_pe_cached.squeeze(1)], dim=-1)
|
||||
k_mqa = k_mqa.unsqueeze(1).expand(-1, num_q_heads, -1)
|
||||
# V: [s_len, kv_lora_rank] (broadcasted to all heads)
|
||||
v_mqa = kv_c_full.unsqueeze(1).expand(-1, num_q_heads, -1)
|
||||
v_mqa = kv_c_cached.unsqueeze(1).expand(-1, num_q_heads, -1)
|
||||
|
||||
# Create custom attention mask for decode path:
|
||||
# - Query tokens can attend to all context tokens
|
||||
@@ -1837,13 +1914,18 @@ def _run_backend_correctness(
|
||||
#######################################################
|
||||
# Prefill path: MHA-style attention with full sequence
|
||||
# Apply kv_b_proj to the full kv_c tensor
|
||||
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_full, kv_b_proj_weight)
|
||||
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_prefill, kv_b_proj_weight)
|
||||
k_nope_full, v_full = kv_nope_full.split([qk_nope_head_dim, v_head_dim], dim=-1)
|
||||
|
||||
# Build attention inputs for full sequence
|
||||
q_mha = torch.cat([q_nope, q_pe], dim=-1) # [q_len, num_heads, total_dim]
|
||||
k_pe_full_expanded = k_pe_full.expand(-1, num_q_heads, -1)
|
||||
k_pe_full_expanded = k_pe_prefill.expand(-1, num_q_heads, -1)
|
||||
k_full = torch.cat([k_nope_full, k_pe_full_expanded], dim=-1)
|
||||
if use_prefill_query_quantization:
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
q_mha = q_mha.to(fp8_dtype).to(dtype)
|
||||
k_full = k_full.to(fp8_dtype).to(dtype)
|
||||
v_full = v_full.to(fp8_dtype).to(dtype)
|
||||
|
||||
# Create custom attention mask:
|
||||
# - Query tokens can attend to all context tokens
|
||||
@@ -1951,6 +2033,13 @@ def _run_backend_correctness(
|
||||
)
|
||||
kv_cache_per_block_size[block_size] = kv_cache
|
||||
|
||||
if adaptive_verification:
|
||||
# Only after the cache is filled: prepopulation reads the context
|
||||
# lengths off the host query lengths.
|
||||
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
|
||||
common_attn_metadata
|
||||
)
|
||||
|
||||
# 4. Run vLLM backends and compare
|
||||
rtol = 1e-2
|
||||
atol = {
|
||||
@@ -2111,6 +2200,8 @@ def test_backend_correctness(
|
||||
_prefill_backend_dimension_params(),
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("trtllm_ragged_backend", ["trtllm-gen", "cute-dsl"])
|
||||
@pytest.mark.parametrize("use_prefill_query_quantization", [False, True])
|
||||
def test_chunked_context_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -2119,8 +2210,26 @@ def test_chunked_context_backend_correctness(
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_cache_dtype: str,
|
||||
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"],
|
||||
use_prefill_query_quantization: bool,
|
||||
):
|
||||
"""Split, packed, and context-free requests match the SDPA reference."""
|
||||
if prefill_backend != MLAPrefillBackendEnum.TRTLLM_RAGGED and (
|
||||
trtllm_ragged_backend == "cute-dsl" or use_prefill_query_quantization
|
||||
):
|
||||
pytest.skip("TRTLLM ragged-specific test configuration")
|
||||
if use_prefill_query_quantization and kv_cache_dtype == "auto":
|
||||
pytest.skip("FP8 prefill queries require an FP8 KV cache")
|
||||
if trtllm_ragged_backend == "cute-dsl":
|
||||
query_dtype = (
|
||||
current_platform.fp8_dtype()
|
||||
if use_prefill_query_quantization
|
||||
else torch.bfloat16
|
||||
)
|
||||
if not is_cute_dsl_mla_prefill_supported(
|
||||
qk_nope_head_dim + 64, v_head_dim, query_dtype
|
||||
):
|
||||
pytest.skip("Unsupported CuTe DSL prefill dimensions or query dtype")
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -2135,4 +2244,32 @@ def test_chunked_context_backend_correctness(
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
chunked_prefill_workspace_size=1024,
|
||||
trtllm_ragged_backend=trtllm_ragged_backend,
|
||||
use_prefill_query_quantization=use_prefill_query_quantization,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
|
||||
)
|
||||
def test_adaptive_varlen_decode_correctness(
|
||||
default_vllm_config, dist_init, workspace_init
|
||||
):
|
||||
"""Device query lengths 1..k+1 against an even host split, as adaptive
|
||||
verification hands them to the target model's attention. TP 8 gives 16
|
||||
heads, the padded Kimi-K3 count."""
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
"spec_decode_ragged_small",
|
||||
"deepseek-ai/DeepSeek-R1",
|
||||
8,
|
||||
"fp8",
|
||||
1.0,
|
||||
1.0,
|
||||
None,
|
||||
128,
|
||||
128,
|
||||
adaptive_verification=True,
|
||||
)
|
||||
|
||||
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
|
||||
stride_0,
|
||||
stride_1,
|
||||
paged_kv_indptr,
|
||||
num_real_tokens,
|
||||
*,
|
||||
KERNEL_BLOCK_SIZE,
|
||||
BLOCK_SIZE,
|
||||
CLAMP_TAIL=False,
|
||||
):
|
||||
# Both block-table strides are passed explicitly: the kernel no longer
|
||||
# assumes the column stride is 1. This stand-in indexes the tensor
|
||||
@@ -145,6 +147,7 @@ def _builder(
|
||||
)
|
||||
),
|
||||
_mtp_decode_qlen=mtp_decode_qlen,
|
||||
_varlen_decode_qlen=0,
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = 576**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=4,
|
||||
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
|
||||
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
|
||||
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = head_dim**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=1,
|
||||
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
|
||||
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
|
||||
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
torch.tensor([global_seq_len], dtype=torch.int32, device=device),
|
||||
)
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=qlen,
|
||||
paged_kv_indptr=torch.tensor(
|
||||
[0, local_kv.shape[0]], dtype=torch.int32, device=device
|
||||
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
|
||||
|
||||
def _adaptive_config(
|
||||
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
enable_adaptive_verification=adaptive,
|
||||
parallel_drafting=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
|
||||
|
||||
def test_varlen_cudagraph_bound_follows_adaptive_verification():
|
||||
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_adaptive_config(), SimpleNamespace()) == 8
|
||||
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
|
||||
# DCP routes plan per-row windows from host lengths.
|
||||
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
|
||||
# Kimi-K3 DSpark draft layers share the target's group; causality is
|
||||
# decided per batch, so the merged non-causal spec keeps the bound.
|
||||
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
|
||||
assert bound(_adaptive_config(), shared_draft_group) == 8
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for KVCacheConfigBuilder resolution."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import Platform
|
||||
from vllm.v1.core.kv_cache_config_builder import (
|
||||
KVCacheConfigBuilder,
|
||||
get_kv_cache_config_builder,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
DefaultKVCacheConfigBuilder,
|
||||
get_kv_cache_groups_from_builder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
|
||||
def _make_vllm_config(supports_grouping: bool = False) -> MagicMock:
|
||||
"""Create a minimal mock VllmConfig for builder resolution tests.
|
||||
|
||||
``supports_grouping`` selects whether the resolved model class declares
|
||||
``SupportsKVCacheGrouping`` (exposing a ``get_kv_cache_groups`` classmethod).
|
||||
"""
|
||||
cfg = MagicMock()
|
||||
cfg.model_config.supports_kv_cache_grouping = supports_grouping
|
||||
cfg.attention_config.hisparse_config = None
|
||||
if supports_grouping:
|
||||
cfg.model_config.registry.resolve_model_cls.return_value = (
|
||||
_GroupedModel,
|
||||
"arch",
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
CUSTOM_PATH = "tests.v1.core.test_kv_cache_config_builder.CustomBuilder"
|
||||
DEFAULT_PATH = "vllm.v1.core.kv_cache_planning.DefaultKVCacheConfigBuilder"
|
||||
|
||||
|
||||
class CustomBuilder(DefaultKVCacheConfigBuilder):
|
||||
"""A test builder subclass."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class _GroupedModel:
|
||||
"""A test model class with a ``get_kv_cache_groups`` classmethod."""
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
|
||||
return []
|
||||
|
||||
|
||||
class _PartialGroupedModel:
|
||||
"""A test model class whose grouping omits layers."""
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
|
||||
|
||||
return [KVCacheGroupSpec(["a"], kv_cache_spec["a"])]
|
||||
|
||||
|
||||
class TestPlatformHookResolution:
|
||||
"""The platform hook selects only the placement builder."""
|
||||
|
||||
def test_default_hook_returns_default_builder(self):
|
||||
cfg = _make_vllm_config()
|
||||
assert Platform.get_kv_cache_config_builder_cls(cfg) == DEFAULT_PATH
|
||||
|
||||
|
||||
class TestBuilderResolution:
|
||||
@patch("vllm.platforms.current_platform")
|
||||
@pytest.mark.parametrize(
|
||||
("builder_cls_path", "expected_cls"),
|
||||
[
|
||||
(None, DefaultKVCacheConfigBuilder),
|
||||
(CUSTOM_PATH, CustomBuilder),
|
||||
],
|
||||
)
|
||||
def test_resolves_hook_selected_builder(
|
||||
self, mock_platform, builder_cls_path, expected_cls
|
||||
):
|
||||
mock_platform.get_kv_cache_config_builder_cls.return_value = (
|
||||
builder_cls_path or DEFAULT_PATH
|
||||
)
|
||||
cfg = _make_vllm_config()
|
||||
assert type(get_kv_cache_config_builder(cfg)) is expected_cls
|
||||
|
||||
@patch("vllm.platforms.current_platform")
|
||||
def test_resolves_fresh_instance_per_call(self, mock_platform):
|
||||
mock_platform.get_kv_cache_config_builder_cls.return_value = CUSTOM_PATH
|
||||
cfg = _make_vllm_config()
|
||||
assert get_kv_cache_config_builder(cfg) is not get_kv_cache_config_builder(cfg)
|
||||
|
||||
def test_public_interface_declares_two_placement_hooks(self):
|
||||
# get_kv_cache_configs lives in kv_cache_planning and is Core-owned;
|
||||
# the builder only declares the placement hooks.
|
||||
assert KVCacheConfigBuilder.__abstractmethods__ == {
|
||||
"get_kv_cache_groups",
|
||||
"get_kv_cache_config_from_groups",
|
||||
}
|
||||
assert issubclass(DefaultKVCacheConfigBuilder, KVCacheConfigBuilder)
|
||||
assert (
|
||||
"pool_bytes_per_block"
|
||||
not in DefaultKVCacheConfigBuilder.__abstractmethods__
|
||||
)
|
||||
assert (
|
||||
"max_memory_usage_bytes"
|
||||
not in DefaultKVCacheConfigBuilder.__abstractmethods__
|
||||
)
|
||||
|
||||
|
||||
class TestModelGroupingPriority:
|
||||
"""A model class with a get_kv_cache_groups classmethod owns grouping."""
|
||||
|
||||
def test_model_classmethod_owns_grouping(self):
|
||||
cfg = _make_vllm_config(supports_grouping=True)
|
||||
assert DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {}) == []
|
||||
|
||||
def test_model_grouping_rejects_hisparse(self):
|
||||
cfg = _make_vllm_config(supports_grouping=True)
|
||||
cfg.attention_config.hisparse_config = object()
|
||||
with pytest.raises(ValueError, match="HiSparse"):
|
||||
DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {})
|
||||
|
||||
|
||||
class TestCoreChecksGrouping:
|
||||
"""Core checks the grouping whether a model or a platform produced it."""
|
||||
|
||||
_SPEC = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
|
||||
)
|
||||
|
||||
def test_model_grouping_is_checked(self):
|
||||
cfg = _make_vllm_config(supports_grouping=True)
|
||||
cfg.model_config.registry.resolve_model_cls.return_value = (
|
||||
_PartialGroupedModel,
|
||||
"arch",
|
||||
)
|
||||
with pytest.raises(AssertionError, match="miss layers"):
|
||||
get_kv_cache_groups_from_builder(
|
||||
DefaultKVCacheConfigBuilder(), cfg, {"a": self._SPEC, "b": self._SPEC}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("groups", "match"),
|
||||
[
|
||||
([KVCacheGroupSpec(["a"], _SPEC)], "miss layers"),
|
||||
(
|
||||
[KVCacheGroupSpec(["a", "b"], _SPEC), KVCacheGroupSpec(["b"], _SPEC)],
|
||||
"overlaps",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_platform_grouping_is_checked(self, groups, match):
|
||||
class FixedGroupsBuilder(DefaultKVCacheConfigBuilder):
|
||||
def get_kv_cache_groups(self, vllm_config, kv_cache_spec):
|
||||
return groups
|
||||
|
||||
with pytest.raises(AssertionError, match=match):
|
||||
get_kv_cache_groups_from_builder(
|
||||
FixedGroupsBuilder(),
|
||||
_make_vllm_config(),
|
||||
{"a": self._SPEC, "b": self._SPEC},
|
||||
)
|
||||
|
||||
|
||||
class TestPlatformCustomPriority:
|
||||
"""A vendor platform can override the hook to customize priority."""
|
||||
|
||||
def test_platform_builder_wins_over_model_declaration(self):
|
||||
class PlatformFirstPlatform(Platform):
|
||||
@classmethod
|
||||
def get_kv_cache_config_builder_cls(cls, vllm_config):
|
||||
return CUSTOM_PATH
|
||||
|
||||
cfg = _make_vllm_config()
|
||||
assert PlatformFirstPlatform.get_kv_cache_config_builder_cls(cfg) == CUSTOM_PATH
|
||||
with patch("vllm.platforms.current_platform", PlatformFirstPlatform):
|
||||
assert isinstance(get_kv_cache_config_builder(cfg), CustomBuilder)
|
||||
|
||||
|
||||
class TestProjectionPreservesGroupMetadata:
|
||||
"""PP projection must carry host_resident/role/enable_kv_transfer."""
|
||||
|
||||
def test_partial_projection_preserves_metadata(self):
|
||||
from vllm.v1.core.kv_cache_planning import _project_kv_cache_groups_to_worker
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupRole,
|
||||
)
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
|
||||
)
|
||||
group = MagicMock()
|
||||
group.layer_names = ["layer1", "layer2", "layer3"]
|
||||
group.kv_cache_spec = spec
|
||||
group.is_eagle_group = False
|
||||
group.host_resident = True
|
||||
group.enable_kv_transfer = False
|
||||
group.role = KVCacheGroupRole.HISPARSE_SOURCE
|
||||
|
||||
projected = _project_kv_cache_groups_to_worker(
|
||||
[group], {"layer1": spec, "layer2": spec}
|
||||
)
|
||||
assert len(projected) == 1
|
||||
p = projected[0]
|
||||
assert p.host_resident is True
|
||||
assert p.enable_kv_transfer is False
|
||||
assert p.role is KVCacheGroupRole.HISPARSE_SOURCE
|
||||
assert p.layer_names == ["layer1", "layer2"]
|
||||
|
||||
|
||||
class TestBytesPerBlockProbe:
|
||||
"""The single-block probe must not be polluted by num_gpu_blocks_override."""
|
||||
|
||||
def test_probe_suppresses_override(self):
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
DefaultKVCacheConfigBuilder,
|
||||
_pool_bytes_per_block,
|
||||
_pool_bytes_per_block_from_builder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.float16
|
||||
)
|
||||
vc = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=100),
|
||||
cache_config=CacheConfig(block_size=16),
|
||||
)
|
||||
resolve_kv_cache_layout(vc, [["BLHNC"]], [spec])
|
||||
groups = [
|
||||
KVCacheGroupSpec(["a"], spec),
|
||||
KVCacheGroupSpec(["b"], spec),
|
||||
]
|
||||
|
||||
pure_c = _pool_bytes_per_block(vc, groups)
|
||||
vc.cache_config.num_gpu_blocks_override = 500
|
||||
probed_c = _pool_bytes_per_block_from_builder(
|
||||
DefaultKVCacheConfigBuilder(), vc, groups
|
||||
)
|
||||
assert probed_c == pure_c
|
||||
assert vc.cache_config.num_gpu_blocks_override == 500
|
||||
|
||||
@pytest.mark.parametrize("declares_pool_bytes", [False, True])
|
||||
def test_several_device_allocations_need_declared_pool_bytes(
|
||||
self, declares_pool_bytes
|
||||
):
|
||||
"""A placement with one allocation per group cannot be sized from one
|
||||
backing; the builder must declare its per-block bytes."""
|
||||
from vllm.v1.core.kv_cache_planning import _pool_bytes_per_block_from_builder
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
|
||||
class PerGroupAllocationBuilder(DefaultKVCacheConfigBuilder):
|
||||
def get_kv_cache_config_from_groups(self, vllm_config, groups, num_blocks):
|
||||
tensors = [
|
||||
KVCacheTensor(
|
||||
size=page * num_blocks,
|
||||
layers=group.layer_names,
|
||||
layer_stride=page * num_blocks,
|
||||
block_stride=page,
|
||||
)
|
||||
for group, page in zip(groups, (64, 32))
|
||||
]
|
||||
return KVCacheConfig(num_blocks, tensors, groups)
|
||||
|
||||
def _get_pool_bytes_per_block(self, vllm_config, groups):
|
||||
return 64 + 32 if declares_pool_bytes else None
|
||||
|
||||
vc = _make_vllm_config()
|
||||
vc.cache_config.num_gpu_blocks_override = None
|
||||
groups = [KVCacheGroupSpec([name], MagicMock()) for name in "ab"]
|
||||
builder = PerGroupAllocationBuilder()
|
||||
if declares_pool_bytes:
|
||||
assert _pool_bytes_per_block_from_builder(builder, vc, groups) == 96
|
||||
else:
|
||||
with pytest.raises(ValueError, match="_get_pool_bytes_per_block"):
|
||||
_pool_bytes_per_block_from_builder(builder, vc, groups)
|
||||
@@ -18,6 +18,7 @@ from vllm.inputs import PromptType
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import EngineCoreRequestType
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.core_client import DPLBAsyncMPClient
|
||||
from vllm.v1.metrics.loggers import StatLoggerBase
|
||||
@@ -336,12 +337,12 @@ async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_late_request_does_not_block_drain():
|
||||
"""A request arriving after pause must not leave the coordinator believing
|
||||
"""A request arriving after pause must not make the coordinator believe
|
||||
the engines are running.
|
||||
|
||||
Paused engines discard START_DP_WAVE, so nothing can report wave
|
||||
completion afterwards; if forwarding the wake also marks the engines as
|
||||
running, drain has no way back and can only end in a timeout.
|
||||
Paused engines start no wave for it, so nothing could report wave
|
||||
completion afterwards; a running flag set on its behalf would leave drain
|
||||
no way back but a timeout.
|
||||
|
||||
MoE only: wave coordination is enabled iff the model is MoE, so a dense
|
||||
model never reaches the coordinator state this exercises.
|
||||
@@ -365,11 +366,9 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
|
||||
# A design that never reports the engines as running would satisfy
|
||||
# every drain assertion below while destroying the signal, so pin it
|
||||
# down first. The front-end sets its own copy optimistically when it
|
||||
# forwards the wake, so sample only after several coordinator
|
||||
# publishes (every 100ms while stats change) have overwritten it.
|
||||
# Rank 0 reports the wave only after its first step, which can outlast
|
||||
# the 2s on a cold device, so wait for the report rather than sample it.
|
||||
# down first. Rank 0 reports the wave only after its first step, which
|
||||
# can outlast the 2s on a cold device, so wait for the report rather
|
||||
# than sample it.
|
||||
await asyncio.sleep(2)
|
||||
assert not long_request.done(), "the warmup request was too short to sample"
|
||||
assert await _poll_flag(engine, True, timeout=30), (
|
||||
@@ -382,24 +381,12 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
await engine.pause_generation(mode="abort")
|
||||
await engine.wait_for_requests_to_drain(drain_timeout=30)
|
||||
|
||||
# Awaiting add_request guarantees the new-request notification has been
|
||||
# sent to the coordinator - the message that used to latch the flag.
|
||||
collector = await engine.add_request(
|
||||
request_id="late",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
# The front-end marks the engines running off the back of that
|
||||
# notification. This is what makes the test non-vacuous: it is the
|
||||
# path that used to leave the coordinator stuck.
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the late request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# It must settle back by itself. Unfixed it never does, because the
|
||||
# paused engines discard the wake and so never report wave completion.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
assert not await _poll_flag(engine, True, timeout=5)
|
||||
|
||||
# The late request was held rather than dropped: it completes on resume.
|
||||
await engine.resume_generation()
|
||||
@@ -409,6 +396,46 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
break
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_request_reaching_rank_after_resume_wakes_peers():
|
||||
"""A request can reach its rank only after resume (e.g. from another API
|
||||
server), with no other work anywhere. That rank must wake its peers itself,
|
||||
or it steps alone. Modelled here by holding the request's ADD on the client
|
||||
side across pause and resume."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
_get_dp_pause_engine_args(expert_parallel=True)
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
client = engine.engine_core
|
||||
params = SamplingParams(max_tokens=5)
|
||||
|
||||
await _consume(engine.generate(DP_PAUSE_PROMPT, params, "warmup"))
|
||||
await engine.pause_generation(mode="abort")
|
||||
assert await _poll_flag(engine, False, timeout=30)
|
||||
|
||||
held: list[tuple] = []
|
||||
send_input = client._send_input
|
||||
|
||||
def hold_add(request_type, request, engine_id=None):
|
||||
if request_type == EngineCoreRequestType.ADD:
|
||||
held.append((request_type, request, engine_id))
|
||||
return asyncio.sleep(0)
|
||||
return send_input(request_type, request, engine_id)
|
||||
|
||||
client._send_input = hold_add
|
||||
task = asyncio.create_task(
|
||||
_consume(engine.generate(DP_PAUSE_PROMPT, params, "late"))
|
||||
)
|
||||
await asyncio.sleep(2)
|
||||
assert held
|
||||
|
||||
await engine.resume_generation()
|
||||
client._send_input = send_input
|
||||
await send_input(*held[0])
|
||||
await asyncio.wait_for(task, timeout=60)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_sleep_late_request_does_not_block_drain():
|
||||
"""The same latch, reached through sleep rather than pause.
|
||||
@@ -441,14 +468,7 @@ async def test_dp_sleep_late_request_does_not_block_drain():
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# Sleeping engines cannot report wave completion, so a coordinator that
|
||||
# marked them running when it forwarded the wake never hears otherwise.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
assert not await _poll_flag(engine, True, timeout=5)
|
||||
|
||||
await engine.wake_up()
|
||||
assert not await engine.is_sleeping()
|
||||
@@ -628,7 +648,7 @@ async def test_dp_pause_keep_race_staggered_engines():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_barrier_request_deadlock():
|
||||
"""Test that start_dp_wave is ignored while paused.
|
||||
"""Test that a request sent while paused does not wake any engine.
|
||||
|
||||
Sequence:
|
||||
1. Pause all engines (PAUSED_ALL).
|
||||
@@ -697,7 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
|
||||
# Yield so generate() preprocessing completes and
|
||||
# add_request_async is called (which, in buggy code,
|
||||
# would send FIRST_REQ and wake engine 1).
|
||||
# would wake engine 1).
|
||||
for _ in range(200):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
@@ -725,7 +745,7 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
for t in mid_barrier_tasks:
|
||||
t.cancel()
|
||||
pytest.fail(
|
||||
"Staggered barrier deadlocked — FIRST_REQ sent while "
|
||||
"Staggered barrier deadlocked — a request sent while "
|
||||
"paused caused collective-op mismatch between engines"
|
||||
)
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.conftest import HfRunner, VllmRunner
|
||||
from tests.utils import (
|
||||
large_gpu_mark,
|
||||
@@ -168,9 +169,16 @@ def test_with_eagle3_spec_decoding(
|
||||
reason=("XPU matmul/attention kernels are not batch-invariant"),
|
||||
)
|
||||
def test_with_ngram_gpu_spec_decoding(
|
||||
vllm_runner: type[VllmRunner], hf_runner: type[HfRunner]
|
||||
vllm_runner: type[VllmRunner],
|
||||
hf_runner: type[HfRunner],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Check ngram accuracy and acceptance across scheduling configurations."""
|
||||
if current_platform.is_rocm():
|
||||
# hipBLASLt picks GEMM solutions by batch shape, so sync and async runs
|
||||
# can flip near-tie greedy tokens and shift ngram acceptance.
|
||||
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1")
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||
ngram_gpu_config = {
|
||||
"method": "ngram_gpu",
|
||||
"num_speculative_tokens": 3,
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.cpu_model
|
||||
@@ -112,3 +113,24 @@ def test_prefix_cache_hit_matches_cold_cache(full_prefill_refs):
|
||||
name_0="cold_cache",
|
||||
name_1="warm_cache",
|
||||
)
|
||||
|
||||
|
||||
def test_prefix_cache_hit_reaches_last_full_block():
|
||||
"""A repeated prompt must hit every full block of the hybrid block size.
|
||||
|
||||
The workers raise the block size to fit the GDN state pages, and the
|
||||
engine core reruns its config checks after its handshake; the raised size
|
||||
must survive them, or the prefill's state checkpoint misses the block
|
||||
boundaries and nothing is reused. One chunk holds the whole prompt.
|
||||
"""
|
||||
llm = _make_llm(enable_prefix_caching=True, max_num_batched_tokens=2048)
|
||||
block_size = llm.llm_engine.vllm_config.cache_config.block_size
|
||||
# Ends mid-block: only the last full block's state can be reused.
|
||||
num_tokens = block_size + block_size // 2
|
||||
assert num_tokens <= 2048
|
||||
prompt = TokensPrompt(prompt_token_ids=[1000 + i for i in range(num_tokens)])
|
||||
sp = SamplingParams(max_tokens=1)
|
||||
hits = [llm.generate(prompt, sp)[0].num_cached_tokens for _ in range(2)]
|
||||
del llm
|
||||
|
||||
assert hits == [0, block_size]
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine import EngineCoreRequest, EngineCoreRequestType
|
||||
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.executor.uniproc_executor import UniProcExecutor
|
||||
@@ -689,6 +689,54 @@ def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
|
||||
assert not core.pending_pause
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_wave,engines_running,pause_state,announces",
|
||||
[
|
||||
pytest.param(3, False, PauseState.UNPAUSED, True, id="current-wave"),
|
||||
pytest.param(2, False, PauseState.UNPAUSED, True, id="stale-wave"),
|
||||
pytest.param(4, False, PauseState.UNPAUSED, True, id="newer-wave"),
|
||||
pytest.param(3, True, PauseState.UNPAUSED, False, id="running"),
|
||||
pytest.param(3, False, PauseState.PAUSED_ALL, False, id="paused"),
|
||||
],
|
||||
)
|
||||
def test_idle_dp_rank_announces_wave_for_new_work(
|
||||
request_wave, engines_running, pause_state, announces
|
||||
):
|
||||
"""An idle rank handed work announces the wave itself, since only engines
|
||||
start waves; a paused rank waits for resume."""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.has_coordinator = True
|
||||
core.current_wave = 3
|
||||
core.engines_running = engines_running
|
||||
core.scheduler = MagicMock(pause_state=pause_state)
|
||||
core.output_queue = MagicMock()
|
||||
|
||||
with patch.object(EngineCore, "add_request"):
|
||||
core.add_request(MagicMock(), request_wave)
|
||||
|
||||
if announces:
|
||||
assert core.engines_running
|
||||
_, outputs = core.output_queue.put_nowait.call_args.args[0]
|
||||
assert outputs.start_wave == max(request_wave, 3)
|
||||
else:
|
||||
core.output_queue.put_nowait.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("paused", [False, True])
|
||||
def test_paused_dp_rank_ignores_start_wave(paused):
|
||||
"""A START_DP_WAVE still in flight when the pause completes must not wake
|
||||
the paused rank."""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.ignore_start_dp_wave = paused
|
||||
core.engine_index = 0
|
||||
core.current_wave = 3
|
||||
core.engines_running = False
|
||||
|
||||
core._handle_client_request(EngineCoreRequestType.START_DP_WAVE, (3, 1))
|
||||
|
||||
assert core.engines_running != paused
|
||||
|
||||
|
||||
def _pausable_engine_core_proc() -> EngineCoreProc:
|
||||
"""A bare EngineCoreProc holding just the state pause_scheduler touches."""
|
||||
core = object.__new__(EngineCoreProc)
|
||||
|
||||
@@ -314,7 +314,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
client.current_wave = 0
|
||||
client.resources = SimpleNamespace(stats_update_task=None)
|
||||
client.stats_update_address = "inproc://scale-down-stats"
|
||||
client.first_req_sock_addr = "inproc://scale-down-first-request"
|
||||
client.scale_sock_addr = "inproc://scale-down-notify"
|
||||
pause_started = asyncio.Event()
|
||||
|
||||
async def pause_scheduler(*args, **kwargs):
|
||||
@@ -326,11 +326,11 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
with (
|
||||
zmq.asyncio.Context() as ctx,
|
||||
ctx.socket(zmq.XPUB) as coordinator,
|
||||
ctx.socket(zmq.PAIR) as first_req_socket,
|
||||
ctx.socket(zmq.PAIR) as scale_socket,
|
||||
):
|
||||
client.ctx = ctx
|
||||
coordinator.bind(client.stats_update_address)
|
||||
first_req_socket.bind(client.first_req_sock_addr)
|
||||
scale_socket.bind(client.scale_sock_addr)
|
||||
client._ensure_stats_update_task()
|
||||
scale_task = None
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,55 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Widen the dummy weight init for the MoRIIO dummy-weight P/D test.
|
||||
|
||||
The dummy loader fills float weights in +-1e-3, which keeps every activation so
|
||||
small that outputs barely depend on the input, so a wrong KV prefix would go
|
||||
unnoticed. The P/D test names ``WideDummyWeights`` as each server's
|
||||
``--worker-extension-cls``. Workers import it before they load the model, which
|
||||
re-registers the ``dummy`` load format in every worker process with a loader
|
||||
that fills 16/32-bit float weights in +-``MORIIO_DUMMY_WEIGHT_RANGE``; 8-bit
|
||||
weights and scales keep the default init. The format keeps the name ``dummy``
|
||||
so every config path that special-cases dummy weights still applies.
|
||||
|
||||
The test process itself never imports this module.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.model_loader import register_model_loader
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
|
||||
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
initialize_single_dummy_weight,
|
||||
)
|
||||
|
||||
_RANGE = float(os.environ["MORIIO_DUMMY_WEIGHT_RANGE"])
|
||||
|
||||
|
||||
@register_model_loader("dummy")
|
||||
class WideDummyModelLoader(DummyModelLoader):
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
# DummyModelLoader.load_weights with a wider init for 16/32-bit floats.
|
||||
for layer in model.modules():
|
||||
info = get_layerwise_info(layer)
|
||||
if info.can_load():
|
||||
self._process_online_quant_layer(layer, info)
|
||||
continue
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
if name in layer._non_persistent_buffers_set:
|
||||
continue
|
||||
if tensor.is_floating_point() and torch.finfo(tensor.dtype).bits >= 16:
|
||||
initialize_single_dummy_weight(
|
||||
tensor, low=-_RANGE, high=_RANGE, name=name
|
||||
)
|
||||
else:
|
||||
initialize_single_dummy_weight(tensor, name=name)
|
||||
|
||||
|
||||
class WideDummyWeights:
|
||||
"""Worker extension that only exists to import this module in workers."""
|
||||
@@ -0,0 +1,736 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Single-node MoRIIO WRITE P/D checks on dummy weights.
|
||||
|
||||
Each case serves a real model architecture with ``--load-format dummy``, widened
|
||||
by ``moriio_wide_dummy`` (see ``DUMMY_WEIGHT_RANGE``): the
|
||||
config keeps the real hidden size, head geometry, MLA ranks and recurrent-state
|
||||
shapes, so the connector registers and transfers exactly the production cache
|
||||
layout, but the layer stack is cut to a few layers that still contain every
|
||||
cache-group kind and the MoE is cut to a handful of experts. Dummy weights are
|
||||
seeded, so the prefill, decode and reference servers hold identical weights.
|
||||
Cases marked ``HMA_off`` run every server with
|
||||
``--disable-hybrid-kv-cache-manager``, so all layers, sliding-window ones
|
||||
included, share a single cache group.
|
||||
|
||||
Prefill runs on one GPU, decode on another, and MoRI moves the KV over xGMI.
|
||||
Either way decode computes the last prompt token itself. For hybrid models
|
||||
prefill computes only the first ``n - 1`` tokens and transfers the KV and the
|
||||
recurrent state after them. For attention-only models prefill computes and
|
||||
transfers all ``n``, and decode's scheduler, seeing the whole prompt cached,
|
||||
recomputes the last token. A direct serve splits the same prompt the same way
|
||||
when the first ``n - 1`` tokens end on a chunk boundary that both servers share,
|
||||
so the prompts are chosen to make that happen: one is a single chunk plus one
|
||||
token, the others are a whole number of hybrid cache blocks plus one token
|
||||
(prefix caching ends chunks on block boundaries) or, for attention-only models,
|
||||
a whole number of chunks plus one token. With the split matched, a correct
|
||||
transfer reproduces the direct serve's first-token log-probabilities exactly,
|
||||
so the comparison needs no noise allowance. The
|
||||
servers compile with Inductor's deterministic mode, since otherwise each one
|
||||
benchmarks its own reduction launch configs and separately started servers
|
||||
can drift apart by a few bf16 logit steps. Guards:
|
||||
|
||||
* decode must report an external prefix-cache hit for every transferred token
|
||||
of every prompt, so the match is not a local recompute;
|
||||
* changing the first half of a prompt must move the direct serve's log-probs by
|
||||
at least ``MIN_CONTEXT_EFFECT``, so a wrong prefix would be visible;
|
||||
* ``test_corrupted_prefill_state_is_detected`` makes prefill compute its KV with
|
||||
a different norm epsilon (same shapes, different values) and requires every
|
||||
prompt to fail the comparison.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[4]
|
||||
_PROXY = (
|
||||
_ROOT / "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
|
||||
)
|
||||
|
||||
# Max allowed difference between centered first-token log-probabilities of the
|
||||
# P/D and direct outputs, over the direct serve's top-k tokens. The split is
|
||||
# matched, so correct transfers produce identical logits; this only absorbs the
|
||||
# fp32 rounding of the log-softmax (~1e-7), far below one bf16 logit step.
|
||||
TOLERANCE = float(os.getenv("MORIIO_DUMMY_E2E_TOL", "1e-5"))
|
||||
# Several bf16 steps of the logits at DUMMY_WEIGHT_RANGE.
|
||||
MIN_CONTEXT_EFFECT = 0.05
|
||||
# Half-width of the uniform dummy init for 16/32-bit float weights, applied in
|
||||
# every server worker by moriio_wide_dummy.py. The loader default (1e-3) shrinks
|
||||
# every activation so far that a context change moves the first-token log-probs
|
||||
# by ~1e-4; at 0.1 it moves them by >0.1 and a corrupted prefill changes the
|
||||
# top-k set entirely.
|
||||
DUMMY_WEIGHT_RANGE = 0.1
|
||||
_WIDE_DUMMY_DIR = Path(__file__).resolve().parent
|
||||
TOP_K = 20
|
||||
MAX_TOKENS = 4
|
||||
MAX_MODEL_LEN = 4096
|
||||
|
||||
# Prefill token budget of every server. Prompts longer than it are chunked, so
|
||||
# the final-chunk save path runs; the short prompt fills exactly one chunk.
|
||||
PREFILL_CHUNK = 299
|
||||
SHORT_PROMPT_LEN = PREFILL_CHUNK + 1
|
||||
# Hybrid blocks per aligned prompt, before the trailing token.
|
||||
ALIGNED_BLOCKS = (1, 2)
|
||||
# Prefill chunks per long attention-only prompt, before the trailing token.
|
||||
ALIGNED_CHUNKS = (2, 4)
|
||||
|
||||
|
||||
def _mori_available() -> bool:
|
||||
try:
|
||||
import mori.io # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not _mori_available(),
|
||||
reason="MoRIIO P/D needs ROCm and MoRI",
|
||||
),
|
||||
pytest.mark.skipif(
|
||||
current_platform.is_rocm() and current_platform.device_count() < 2,
|
||||
reason="needs one GPU for prefill and one for decode",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model architectures
|
||||
|
||||
|
||||
def _kimi_linear_overrides(cfg) -> dict[str, Any]:
|
||||
# 3 KDA layers then 1 MLA layer; layer 0 stays dense (first_k_dense_replace).
|
||||
linear = dict(cfg.linear_attn_config)
|
||||
linear["kda_layers"] = [1, 2, 3]
|
||||
linear["full_attn_layers"] = [4]
|
||||
groups = max(1, cfg.num_expert_group)
|
||||
return {
|
||||
"num_hidden_layers": 4,
|
||||
"linear_attn_config": linear,
|
||||
"num_experts": 2 * groups,
|
||||
"num_experts_per_token": 2,
|
||||
"topk_group": min(cfg.topk_group, groups),
|
||||
}
|
||||
|
||||
|
||||
def _qwen3_5_moe_overrides(cfg) -> dict[str, Any]:
|
||||
# 3 Gated DeltaNet layers then 1 full-attention layer.
|
||||
return {
|
||||
"text_config": {
|
||||
"num_hidden_layers": 4,
|
||||
"layer_types": ["linear_attention"] * 3 + ["full_attention"],
|
||||
"num_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _qwen3_dense_overrides(cfg) -> dict[str, Any]:
|
||||
# Attention-only: decode gets the KV for the whole prompt.
|
||||
return {"num_hidden_layers": 4}
|
||||
|
||||
|
||||
def _deepseek_v3_overrides(cfg) -> dict[str, Any]:
|
||||
# MLA attention-only: layer 0 dense, layer 1 MoE, no MTP layer.
|
||||
return {
|
||||
"num_hidden_layers": 2,
|
||||
"first_k_dense_replace": 1,
|
||||
"n_routed_experts": 8,
|
||||
"num_experts_per_tok": 2,
|
||||
"n_group": 1,
|
||||
"topk_group": 1,
|
||||
"num_nextn_predict_layers": 0,
|
||||
}
|
||||
|
||||
|
||||
def _gemma3_overrides(cfg) -> dict[str, Any]:
|
||||
# Five sliding-window layers then one global layer, as in the real
|
||||
# 5:1 pattern. Without the hybrid KV cache manager vLLM gives the sliding
|
||||
# layers full-attention cache specs, so all six share one group.
|
||||
return {
|
||||
"num_hidden_layers": 6,
|
||||
"layer_types": ["sliding_attention"] * 5 + ["full_attention"],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArchCase:
|
||||
model: str
|
||||
overrides: Callable[[Any], dict[str, Any]]
|
||||
# Norm-epsilon override that perturbs every computed KV/state value
|
||||
# without changing a single shape. Used by the corruption control.
|
||||
corrupt_overrides: dict[str, Any]
|
||||
env: dict[str, str] = field(default_factory=dict)
|
||||
args: tuple[str, ...] = ()
|
||||
# Has recurrent-state (mamba-style) layers, so prefill stops at n - 1.
|
||||
hybrid: bool = True
|
||||
|
||||
|
||||
ARCHS: dict[str, ArchCase] = {
|
||||
"kimi_linear": ArchCase(
|
||||
model="moonshotai/Kimi-Linear-48B-A3B-Instruct",
|
||||
overrides=_kimi_linear_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "1"},
|
||||
args=("--trust-remote-code", "--mamba-ssm-cache-dtype", "float32"),
|
||||
),
|
||||
"qwen3_5_moe": ArchCase(
|
||||
model="Qwen/Qwen3.5-35B-A3B",
|
||||
overrides=_qwen3_5_moe_overrides,
|
||||
corrupt_overrides={"text_config": {"rms_norm_eps": 1.0}},
|
||||
# AITER's GDN conv-update kernel rejects the DS conv-state layout that
|
||||
# MoRIIO's recurrent-state transfer requires.
|
||||
env={"VLLM_ROCM_USE_AITER": "0"},
|
||||
args=("--language-model-only",),
|
||||
),
|
||||
"qwen3_dense": ArchCase(
|
||||
model="Qwen/Qwen3-32B",
|
||||
overrides=_qwen3_dense_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "0"},
|
||||
hybrid=False,
|
||||
),
|
||||
"deepseek_v3": ArchCase(
|
||||
model="deepseek-ai/DeepSeek-V3",
|
||||
overrides=_deepseek_v3_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "1"},
|
||||
hybrid=False,
|
||||
),
|
||||
# Text-only Gemma 3. The multimodal Gemma3ForConditionalGeneration takes
|
||||
# the inputs_embeds path on KV consumers (enable_mm_embeds is forced on
|
||||
# there) and computes differently from a direct serve without any transfer.
|
||||
"gemma3": ArchCase(
|
||||
model="google/gemma-3-1b-it",
|
||||
overrides=_gemma3_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "0"},
|
||||
hybrid=False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _arch_overrides(arch: ArchCase) -> dict[str, Any]:
|
||||
from vllm.transformers_utils.config import get_config
|
||||
|
||||
return arch.overrides(get_config(arch.model, trust_remote_code=True))
|
||||
|
||||
|
||||
def _merge(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]:
|
||||
merged = dict(base)
|
||||
for key, value in extra.items():
|
||||
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key] = _merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serving
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Case:
|
||||
arch: str
|
||||
attention_backend: str
|
||||
prefill_graph: str
|
||||
decode_graph: str
|
||||
# Run every server of the case without the hybrid KV cache manager.
|
||||
hma_off: bool = False
|
||||
|
||||
@property
|
||||
def extra_args(self) -> tuple[str, ...]:
|
||||
return ("--disable-hybrid-kv-cache-manager",) if self.hma_off else ()
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return (
|
||||
f"{self.arch}-{self.attention_backend}-"
|
||||
f"P_{self.prefill_graph}-D_{self.decode_graph}"
|
||||
+ ("-HMA_off" if self.hma_off else "")
|
||||
)
|
||||
|
||||
|
||||
_DEFAULT = "FULL_AND_PIECEWISE"
|
||||
CASES = [
|
||||
# Kimi-Linear: KDA + MLA groups.
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT),
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", "NONE", "NONE"),
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", "PIECEWISE", "PIECEWISE"),
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, "FULL_DECODE_ONLY"),
|
||||
Case("kimi_linear", "TRITON_MLA", _DEFAULT, _DEFAULT),
|
||||
# Qwen3.5: Gated DeltaNet + full-attention groups.
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, _DEFAULT),
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", "NONE", "NONE"),
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", "PIECEWISE", "PIECEWISE"),
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, "FULL_DECODE_ONLY"),
|
||||
# ROCM_ATTN keeps a (2, num_blocks, ...) KV layout and opts out of KV
|
||||
# connectors, so the second ROCm attention backend is the unified one.
|
||||
Case("qwen3_5_moe", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
|
||||
# Attention-only: prefill transfers the whole prompt.
|
||||
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT),
|
||||
Case("qwen3_dense", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
|
||||
# Without the hybrid KV cache manager: one cache group per model.
|
||||
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
|
||||
Case("deepseek_v3", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT, hma_off=True),
|
||||
Case("gemma3", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
|
||||
]
|
||||
|
||||
|
||||
def _graph_args(mode: str) -> list[str]:
|
||||
if mode == "NONE":
|
||||
return ["--enforce-eager"]
|
||||
# Inductor otherwise picks reduction launch configs by timing them, so each
|
||||
# separately compiled server may sum in a different order and drift by a
|
||||
# few bf16 logit steps; deterministic mode fixes the choice.
|
||||
config = {
|
||||
"cudagraph_mode": mode,
|
||||
"inductor_compile_config": {"deterministic": True},
|
||||
}
|
||||
return ["--compilation-config", json.dumps(config)]
|
||||
|
||||
|
||||
def _visible_devices() -> tuple[str, list[str]]:
|
||||
selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
|
||||
selector = next((k for k in selectors if k in os.environ), selectors[0])
|
||||
visible = os.getenv(selector)
|
||||
devices = (
|
||||
visible.split(",")
|
||||
if visible
|
||||
else [str(i) for i in range(current_platform.device_count())]
|
||||
)
|
||||
return selector, devices
|
||||
|
||||
|
||||
def _device_env(selector: str, device: str) -> dict[str, str]:
|
||||
# vLLM mirrors HIP_VISIBLE_DEVICES into CUDA_VISIBLE_DEVICES at import, so
|
||||
# the test process carries both; pin every selector or the server refuses
|
||||
# the mismatch. ROCR narrows the set first, so HIP/CUDA then index into it.
|
||||
env = {selector: device}
|
||||
for key in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
|
||||
if key != selector:
|
||||
env[key] = "0" if selector == "ROCR_VISIBLE_DEVICES" else device
|
||||
return env
|
||||
|
||||
|
||||
# KV consumers force enable_mm_embeds on, which moves multimodal-capable models
|
||||
# (Qwen3.5) from input_ids to inputs_embeds, but the AOT compile-cache key does
|
||||
# not cover it, so a consumer can load a producer's graph and crash in
|
||||
# profile_run. Keep each role's compile cache apart until
|
||||
# vllm-project/vllm#54642 lands (vllm-project/vllm#50891).
|
||||
_CACHE_ROOT = Path(tempfile.mkdtemp(prefix="moriio_dummy_e2e_cache_"))
|
||||
atexit.register(shutil.rmtree, _CACHE_ROOT, ignore_errors=True)
|
||||
|
||||
|
||||
def _server(
|
||||
arch: ArchCase,
|
||||
overrides: dict[str, Any],
|
||||
backend: str,
|
||||
graph: str,
|
||||
device: str,
|
||||
selector: str,
|
||||
*,
|
||||
port: int | None = None,
|
||||
extra_args: tuple[str, ...] = (),
|
||||
kv_transfer_config: dict[str, Any] | None = None,
|
||||
) -> RemoteOpenAIServer:
|
||||
port = port or get_open_port()
|
||||
args = [
|
||||
*arch.args,
|
||||
*extra_args,
|
||||
"--port",
|
||||
str(port),
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--worker-extension-cls",
|
||||
"moriio_wide_dummy.WideDummyWeights",
|
||||
"--hf-overrides",
|
||||
json.dumps(overrides),
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--attention-backend",
|
||||
backend,
|
||||
"--max-model-len",
|
||||
str(MAX_MODEL_LEN),
|
||||
"--max-num-batched-tokens",
|
||||
str(PREFILL_CHUNK),
|
||||
"--max-num-seqs",
|
||||
"16",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(2 * 1024**3),
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--enable-prefix-caching",
|
||||
*_graph_args(graph),
|
||||
]
|
||||
role = "direct"
|
||||
if kv_transfer_config is not None:
|
||||
args += ["--kv-transfer-config", json.dumps(kv_transfer_config)]
|
||||
role = kv_transfer_config["kv_role"]
|
||||
env = {
|
||||
"PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"],
|
||||
**_device_env(selector, device),
|
||||
"VLLM_HOST_IP": "127.0.0.1",
|
||||
"VLLM_PORT": str(get_open_port()),
|
||||
"VLLM_WORKER_MULTIPROC_METHOD": "spawn",
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT": "DS",
|
||||
"VLLM_CACHE_ROOT": str(_CACHE_ROOT / role),
|
||||
# Only this directory, so the source tree cannot shadow installed vllm.
|
||||
"PYTHONPATH": os.pathsep.join(
|
||||
filter(None, [str(_WIDE_DUMMY_DIR), os.environ.get("PYTHONPATH")])
|
||||
),
|
||||
"MORIIO_DUMMY_WEIGHT_RANGE": str(DUMMY_WEIGHT_RANGE),
|
||||
**arch.env,
|
||||
}
|
||||
return RemoteOpenAIServer(
|
||||
arch.model, args, env_dict=env, auto_port=False, max_wait_seconds=900
|
||||
)
|
||||
|
||||
|
||||
def _moriio_config(role: str, http_port: int, ping_port: int) -> dict[str, Any]:
|
||||
return {
|
||||
"kv_connector": "MoRIIOConnector",
|
||||
"kv_role": role,
|
||||
"kv_connector_extra_config": {
|
||||
"backend": "xgmi",
|
||||
"read_mode": False,
|
||||
"host_ip": "127.0.0.1",
|
||||
"proxy_ip": "127.0.0.1",
|
||||
"proxy_ping_port": str(ping_port),
|
||||
"http_port": str(http_port),
|
||||
"handshake_port": str(get_open_port()),
|
||||
"notify_port": str(get_open_port()),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _proxy(ping_port: int) -> Iterator[str]:
|
||||
# Free ports for both sockets, so cases from parallel shards on one host
|
||||
# do not collide.
|
||||
port = get_open_port()
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
str(_PROXY),
|
||||
"--port",
|
||||
str(port),
|
||||
"--service-discovery-port",
|
||||
str(ping_port),
|
||||
],
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
assert process.poll() is None, "MoRIIO proxy exited during startup"
|
||||
try:
|
||||
requests.get(f"http://127.0.0.1:{port}/", timeout=1)
|
||||
break
|
||||
except requests.ConnectionError:
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise TimeoutError("MoRIIO proxy did not start")
|
||||
yield f"http://127.0.0.1:{port}"
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=10)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _pd_pair(
|
||||
arch: ArchCase,
|
||||
prefill_overrides: dict[str, Any],
|
||||
decode_overrides: dict[str, Any],
|
||||
case: Case,
|
||||
) -> Iterator[tuple[str, RemoteOpenAIServer]]:
|
||||
selector, devices = _visible_devices()
|
||||
assert len(devices) >= 2, "needs one GPU for prefill and one for decode"
|
||||
prefill_port, decode_port, ping_port = (get_open_port() for _ in range(3))
|
||||
with _proxy(ping_port) as proxy_url, contextlib.ExitStack() as stack:
|
||||
servers: list[RemoteOpenAIServer] = []
|
||||
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
|
||||
with ThreadPoolExecutor(2) as pool:
|
||||
futures = [
|
||||
pool.submit(
|
||||
_server,
|
||||
arch,
|
||||
prefill_overrides,
|
||||
case.attention_backend,
|
||||
case.prefill_graph,
|
||||
devices[0],
|
||||
selector,
|
||||
port=prefill_port,
|
||||
extra_args=case.extra_args,
|
||||
kv_transfer_config=_moriio_config(
|
||||
"kv_producer", prefill_port, ping_port
|
||||
),
|
||||
),
|
||||
pool.submit(
|
||||
_server,
|
||||
arch,
|
||||
decode_overrides,
|
||||
case.attention_backend,
|
||||
case.decode_graph,
|
||||
devices[1],
|
||||
selector,
|
||||
port=decode_port,
|
||||
extra_args=case.extra_args,
|
||||
kv_transfer_config=_moriio_config(
|
||||
"kv_consumer", decode_port, ping_port
|
||||
),
|
||||
),
|
||||
]
|
||||
for future in futures:
|
||||
servers.append(future.result())
|
||||
_wait_for_registration(proxy_url, arch.model)
|
||||
yield proxy_url, servers[1]
|
||||
|
||||
|
||||
def _wait_for_registration(proxy_url: str, model: str) -> None:
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
# The proxy hands decode max_tokens - 1 (prefill emits the first
|
||||
# token), so the probe needs at least 2 for decode to accept it.
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{proxy_url}/v1/completions",
|
||||
json={"model": model, "prompt": [1000, 1001], "max_tokens": 2},
|
||||
timeout=300,
|
||||
)
|
||||
# The proxy streams the decode body, so a decode-side error can
|
||||
# surface as a truncated response rather than a status code.
|
||||
if response.status_code == 200 and response.json()["choices"]:
|
||||
return
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise TimeoutError("prefill/decode never registered with the MoRIIO proxy")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Measurement
|
||||
|
||||
|
||||
def _metric(server: RemoteOpenAIServer, name: str) -> list[Any]:
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
response = requests.get(server.url_for("metrics"), timeout=10)
|
||||
response.raise_for_status()
|
||||
return [
|
||||
sample
|
||||
for family in text_string_to_metric_families(response.text)
|
||||
for sample in family.samples
|
||||
if sample.name == name
|
||||
]
|
||||
|
||||
|
||||
def _block_size(server: RemoteOpenAIServer) -> int:
|
||||
# Hybrid models pad the attention block to the recurrent-state page, so the
|
||||
# effective size is only known once the server has started.
|
||||
sizes = {
|
||||
int(sample.labels["block_size"])
|
||||
for sample in _metric(server, "vllm:cache_config_info")
|
||||
}
|
||||
assert len(sizes) == 1, f"expected one cache block size, got {sizes}"
|
||||
return sizes.pop()
|
||||
|
||||
|
||||
def _prompt_lengths(arch: ArchCase, server: RemoteOpenAIServer) -> list[int]:
|
||||
# Hybrid chunks end on cache-block boundaries; attention-only chunks end on
|
||||
# the prefill budget, so the last token is computed on its own either way.
|
||||
if arch.hybrid:
|
||||
unit, counts = _block_size(server), ALIGNED_BLOCKS
|
||||
else:
|
||||
unit, counts = PREFILL_CHUNK, ALIGNED_CHUNKS
|
||||
aligned = [
|
||||
k * unit + 1 for k in counts if k * unit + 1 + MAX_TOKENS <= MAX_MODEL_LEN
|
||||
]
|
||||
assert aligned, f"alignment unit {unit} leaves no aligned prompt"
|
||||
return [SHORT_PROMPT_LEN, *aligned]
|
||||
|
||||
|
||||
def _prompts(lengths: list[int]) -> list[list[int]]:
|
||||
rng = random.Random(0)
|
||||
return [[rng.randrange(1000, 30000) for _ in range(n)] for n in lengths]
|
||||
|
||||
|
||||
def _first_step(url: str, model: str, prompt: list[int]) -> dict[str, float]:
|
||||
response = requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"temperature": 0,
|
||||
"logprobs": TOP_K,
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
timeout=600,
|
||||
)
|
||||
response.raise_for_status()
|
||||
top = response.json()["choices"][0]["logprobs"]["top_logprobs"]
|
||||
assert top, "server returned no logprobs"
|
||||
return top[0]
|
||||
|
||||
|
||||
def _distance(ref: dict[str, float], got: dict[str, float]) -> float:
|
||||
"""Max difference of centered log-probs over the shared top-k tokens.
|
||||
|
||||
Centering removes the log-sum-exp offset, leaving the logit differences the
|
||||
KV actually determines.
|
||||
"""
|
||||
common = sorted(ref.keys() & got.keys())
|
||||
if len(common) < len(ref) // 2:
|
||||
return float("inf")
|
||||
ref_mean = sum(ref[t] for t in common) / len(common)
|
||||
got_mean = sum(got[t] for t in common) / len(common)
|
||||
return max(abs((ref[t] - ref_mean) - (got[t] - got_mean)) for t in common)
|
||||
|
||||
|
||||
def _external_hits(server: RemoteOpenAIServer) -> float:
|
||||
samples = _metric(server, "vllm:external_prefix_cache_hits_total")
|
||||
return sum(sample.value for sample in samples)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Reference:
|
||||
prompts: list[list[int]]
|
||||
logprobs: list[dict[str, float]]
|
||||
hybrid: bool
|
||||
|
||||
@property
|
||||
def expected_external_hits(self) -> int:
|
||||
# Hybrid prefill stops one token short; attention-only prefill hands
|
||||
# decode the whole prompt.
|
||||
return sum(len(p) - int(self.hybrid) for p in self.prompts)
|
||||
|
||||
|
||||
def _reference(arch: ArchCase, overrides: dict[str, Any], case: Case) -> Reference:
|
||||
"""Direct serve; also checks that the comparison can see a wrong prefix."""
|
||||
selector, devices = _visible_devices()
|
||||
# Prefill graphs run the chunks and the decode-only last step runs the same
|
||||
# captured graph in every decode mode used here, so the prefill mode
|
||||
# reproduces both halves of the P/D computation.
|
||||
with _server(
|
||||
arch,
|
||||
overrides,
|
||||
case.attention_backend,
|
||||
case.prefill_graph,
|
||||
devices[0],
|
||||
selector,
|
||||
extra_args=case.extra_args,
|
||||
) as server:
|
||||
url = server.url_root
|
||||
lengths = _prompt_lengths(arch, server)
|
||||
prompts = _prompts(lengths)
|
||||
refs = [_first_step(url, arch.model, p) for p in prompts]
|
||||
# Same tail, different early context: the comparison has to see it.
|
||||
for prompt, ref in zip(prompts, refs):
|
||||
altered = [t + 1 for t in prompt[: len(prompt) // 2]]
|
||||
altered += prompt[len(prompt) // 2 :]
|
||||
moved = _distance(ref, _first_step(url, arch.model, altered))
|
||||
print(f"[sensitivity] len={len(prompt)} context-change distance={moved}")
|
||||
assert moved >= MIN_CONTEXT_EFFECT, (
|
||||
f"first-token log-probs barely depend on the context (moved "
|
||||
f"{moved:.4g} for len {len(prompt)}); a wrong prefix would go "
|
||||
"unnoticed"
|
||||
)
|
||||
# Determinism of the reference itself (served from its prefix cache).
|
||||
again = _first_step(url, arch.model, prompts[-1])
|
||||
print(f"[noise] repeat distance={_distance(refs[-1], again)}")
|
||||
return Reference(prompts, refs, arch.hybrid)
|
||||
|
||||
|
||||
def _pd_distances(
|
||||
arch: ArchCase,
|
||||
prefill_overrides: dict[str, Any],
|
||||
decode_overrides: dict[str, Any],
|
||||
case: Case,
|
||||
ref: Reference,
|
||||
) -> tuple[list[float], float]:
|
||||
with _pd_pair(arch, prefill_overrides, decode_overrides, case) as (
|
||||
proxy_url,
|
||||
decode,
|
||||
):
|
||||
hits_before = _external_hits(decode)
|
||||
distances = [
|
||||
_distance(expected, _first_step(proxy_url, arch.model, prompt))
|
||||
for prompt, expected in zip(ref.prompts, ref.logprobs)
|
||||
]
|
||||
hits = _external_hits(decode) - hits_before
|
||||
lengths = [len(p) for p in ref.prompts]
|
||||
print(f"[pd] {case.id} lens={lengths} distances={distances} external_hits={hits}")
|
||||
return distances, hits
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id)
|
||||
def test_write_matches_direct_serve(case: Case):
|
||||
arch = ARCHS[case.arch]
|
||||
overrides = _arch_overrides(arch)
|
||||
ref = _reference(arch, overrides, case)
|
||||
distances, hits = _pd_distances(arch, overrides, overrides, case, ref)
|
||||
|
||||
# Every transferred token of every prompt must come from prefill.
|
||||
assert hits == ref.expected_external_hits, (
|
||||
f"decode reported {hits} external prefix-cache hits, expected "
|
||||
f"{ref.expected_external_hits}; it did not use all the transferred KV"
|
||||
)
|
||||
lengths = [len(p) for p in ref.prompts]
|
||||
bad = {n: d for n, d in zip(lengths, distances) if not d <= TOLERANCE}
|
||||
assert not bad, f"P/D first-token log-probs differ from direct serve: {bad}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arch_name", sorted(ARCHS))
|
||||
def test_corrupted_prefill_state_is_detected(arch_name: str):
|
||||
"""Negative control: KV computed with different values must fail."""
|
||||
arch = ARCHS[arch_name]
|
||||
default = next(c for c in CASES if c.arch == arch_name)
|
||||
case = Case(
|
||||
arch_name, default.attention_backend, _DEFAULT, _DEFAULT, default.hma_off
|
||||
)
|
||||
overrides = _arch_overrides(arch)
|
||||
ref = _reference(arch, overrides, case)
|
||||
corrupted = _merge(overrides, arch.corrupt_overrides)
|
||||
distances, hits = _pd_distances(arch, corrupted, overrides, case, ref)
|
||||
|
||||
assert hits == ref.expected_external_hits, (
|
||||
"decode did not use the transferred KV; the control proves nothing"
|
||||
)
|
||||
assert all(d > TOLERANCE for d in distances), (
|
||||
f"P/D outputs with wrongly computed prefill KV still matched the "
|
||||
f"reference (distances {distances}); the comparison cannot detect "
|
||||
"transfer corruption"
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("endpoint", ["/v1/completions", "/v1/chat/completions"])
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_response_media_type_matches_payload(monkeypatch, endpoint, stream):
|
||||
"""Clients must select SSE parsing for streaming completion responses."""
|
||||
path = (
|
||||
Path(__file__).parents[4]
|
||||
/ "examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py"
|
||||
)
|
||||
proxy = runpy.run_path(str(path))["_handle_completions"].__globals__
|
||||
app = proxy["app"]
|
||||
app.state.ready = asyncio.Event()
|
||||
app.state.ready.set()
|
||||
app.state.prefill_iterator = itertools.cycle([({}, 0)])
|
||||
app.state.decode_iterator = itertools.cycle([0])
|
||||
app.state.decode_clients = [{}]
|
||||
payload = b'{"choices":[{"text":"hello"}]}'
|
||||
body = b"data: " + payload + b"\n\ndata: [DONE]\n\n" if stream else payload
|
||||
|
||||
async def decode_response(*args, **kwargs):
|
||||
yield body
|
||||
|
||||
monkeypatch.setitem(proxy, "send_request_to_service", AsyncMock())
|
||||
monkeypatch.setitem(proxy, "stream_service_response", decode_response)
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://proxy"
|
||||
) as client:
|
||||
response = await client.post(endpoint, json={"stream": stream})
|
||||
|
||||
expected_type = "text/event-stream" if stream else "application/json"
|
||||
assert response.headers["content-type"].split(";")[0] == expected_type
|
||||
assert response.content == body
|
||||
@@ -146,6 +146,7 @@ def _write_consumer_scheduler_for_finished_request(tp_size: int = 2):
|
||||
scheduler.is_producer = False
|
||||
scheduler.mode = MoRIIOMode.WRITE
|
||||
scheduler.tp_size = tp_size
|
||||
scheduler._has_mamba = False
|
||||
scheduler._reqs_need_recv = {}
|
||||
scheduler.unmap_request_id = MagicMock()
|
||||
return scheduler
|
||||
@@ -158,6 +159,10 @@ def _write_producer_scheduler(block_size: int = 1) -> Any:
|
||||
scheduler.mode = MoRIIOMode.WRITE
|
||||
scheduler.is_producer = True
|
||||
scheduler.block_size = block_size
|
||||
scheduler._has_mamba = False
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=object())]
|
||||
)
|
||||
scheduler.blocks_per_sw = [0]
|
||||
scheduler._group_block_sizes = [block_size]
|
||||
scheduler.transfer_id_to_request_id = {}
|
||||
@@ -944,6 +949,7 @@ def test_clamp_to_prompt_blocks_clamps_each_group_by_its_own_block_size():
|
||||
# their own prompt-block count; a sliding-window group is left untouched.
|
||||
scheduler = MoRIIOConnectorScheduler.__new__(MoRIIOConnectorScheduler)
|
||||
scheduler.blocks_per_sw = [0, 0, 3]
|
||||
scheduler._has_mamba = False
|
||||
scheduler._group_block_sizes = [16, 32, 16]
|
||||
|
||||
req = SimpleNamespace(num_prompt_tokens=40)
|
||||
|
||||
@@ -12,12 +12,13 @@ wiring.
|
||||
import importlib
|
||||
from dataclasses import asdict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.output import KVConnectorBlockState, SchedulerOutput
|
||||
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs
|
||||
|
||||
@@ -52,6 +53,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
|
||||
attributes read by the method under test are set."""
|
||||
|
||||
def __init__(self, **attrs):
|
||||
self.mode = MoRIIOMode.READ
|
||||
self._mamba_group_ids: list[int] = []
|
||||
self._attn_group_ids: list[int] = [0]
|
||||
self._num_ssm_scratch_blocks = 0
|
||||
@@ -287,7 +289,8 @@ def test_scheduler_rejects_mamba_group_without_two_states():
|
||||
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
|
||||
|
||||
|
||||
def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
|
||||
@pytest.mark.parametrize("read_mode", [True, False])
|
||||
def test_scheduler_rejects_non_dspark_speculative_hybrid(read_mode):
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
@@ -298,7 +301,9 @@ def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(speculative_config=_spec_config("ngram")),
|
||||
_gate_vllm_config(
|
||||
read_mode=read_mode, speculative_config=_spec_config("ngram")
|
||||
),
|
||||
"engine",
|
||||
config,
|
||||
)
|
||||
@@ -336,14 +341,16 @@ def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True):
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
@pytest.mark.parametrize("read_mode", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)]
|
||||
)
|
||||
def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
dcp_size, dcp_sharded, expected_tail
|
||||
dcp_size, dcp_sharded, expected_tail, read_mode
|
||||
):
|
||||
config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded)
|
||||
vllm_config = _gate_vllm_config(
|
||||
read_mode=read_mode,
|
||||
speculative_config=_spec_config("dspark"),
|
||||
block_size=4,
|
||||
num_lookahead_tokens=7,
|
||||
@@ -384,19 +391,234 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_hybrid_write():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
|
||||
],
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_clamps_attention_and_excludes_scratch_slots():
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
assert sched._clamp_to_prompt_blocks(
|
||||
SimpleNamespace(num_prompt_tokens=9), [[10, 11, 12, 13], [1, 2, 3, 4]]
|
||||
) == [[11], [1, 2]]
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False), "engine", config
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_resolves_current_state_instead_of_cached_block_ids():
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
state = KVConnectorBlockState(
|
||||
req_ids={"req"},
|
||||
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
|
||||
boundary_state_offloads={},
|
||||
)
|
||||
assert sched._get_write_blocks(
|
||||
SimpleNamespace(request_id="req", num_prompt_tokens=9),
|
||||
[[10, 11, 12], [1]],
|
||||
SimpleNamespace(kv_connector_block_state=state),
|
||||
) == [[51], [31, 32]]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
@pytest.mark.parametrize("external_tokens", [0, 9])
|
||||
def test_hybrid_write_notifies_grouped_slots_or_releases_a_complete_local_hit(
|
||||
external_tokens,
|
||||
):
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False), "engine", config
|
||||
)
|
||||
sched.send_notify_block = Mock()
|
||||
sched._send_transfer_release = Mock()
|
||||
req = _mk_request(
|
||||
list(range(10)),
|
||||
params={
|
||||
"do_remote_prefill": True,
|
||||
"transfer_id": "tx",
|
||||
"remote_host": "127.0.0.2",
|
||||
"remote_notify_port": 7000,
|
||||
},
|
||||
)
|
||||
req.request_id = "req"
|
||||
sched.update_state_after_alloc(
|
||||
req, _FakeBlocks(([10, 11, 12], [1, 2, 3])), external_tokens
|
||||
)
|
||||
if external_tokens:
|
||||
sched.send_notify_block.assert_called_once_with(
|
||||
req_id="req",
|
||||
transfer_id="tx",
|
||||
block_notify_list=[[10], [1, 2, 3]],
|
||||
host="127.0.0.2",
|
||||
port=7000,
|
||||
)
|
||||
sched._send_transfer_release.assert_not_called()
|
||||
else:
|
||||
sched.send_notify_block.assert_not_called()
|
||||
sched._send_transfer_release.assert_called_once_with("tx", "127.0.0.2", 7000)
|
||||
assert not req.kv_transfer_params["do_remote_prefill"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_keeps_separate_draft_groups_and_their_block_spans():
|
||||
config = _dspark_wrapped_attention_config()
|
||||
# A disabled cache is absent from the wire. Target and draft may have
|
||||
# different token spans, and neither can borrow the other's block IDs.
|
||||
draft = SimpleNamespace(
|
||||
enable_kv_transfer=True,
|
||||
kv_cache_spec=FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
dcp_sharded=False,
|
||||
),
|
||||
)
|
||||
disabled = SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object())
|
||||
config.kv_cache_groups.insert(1, disabled)
|
||||
config.kv_cache_groups.append(draft)
|
||||
config.transfer_groups = tuple(
|
||||
g for g in config.kv_cache_groups if g.enable_kv_transfer
|
||||
)
|
||||
config.select_transfer_block_ids = lambda blocks: (blocks[0], blocks[2], blocks[3])
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(
|
||||
read_mode=False, dcp_size=2, speculative_config=_spec_config("dspark")
|
||||
),
|
||||
"engine",
|
||||
config,
|
||||
)
|
||||
req = _mk_request(
|
||||
list(range(9)),
|
||||
params={
|
||||
"do_remote_decode": True,
|
||||
"transfer_id": "tx",
|
||||
},
|
||||
)
|
||||
req.request_id = "req"
|
||||
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
blocks = ([10, 11, 12, 13], [99], [20, 21, 22], [30, 31, 32, 33])
|
||||
expected = [[11], [20, 21], [30, 31, 32]]
|
||||
assert sched._clamp_to_prompt_blocks(req, blocks) == expected
|
||||
# Exercise the connector entry point too: completion must not flatten
|
||||
# a multi-group WRITE payload through READ's single-attention helper.
|
||||
delay_free, params = _FakeConnector(sched).request_finished_all_groups(req, blocks)
|
||||
assert delay_free
|
||||
assert params["remote_block_ids"] == expected
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
@pytest.mark.parametrize(
|
||||
"blocks,delay_free",
|
||||
[(([0, 51, 52, 53], [31, 32, 33]), True), (([], []), False)],
|
||||
)
|
||||
def test_hybrid_write_request_finished_defers_free_only_with_blocks_to_send(
|
||||
blocks, delay_free
|
||||
):
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
sched.is_producer = True
|
||||
req = _mk_request(
|
||||
list(range(9)), params={"do_remote_decode": True, "transfer_id": "tx"}
|
||||
)
|
||||
req.request_id = "req"
|
||||
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
|
||||
assert sched.request_finished(req, blocks)[0] is delay_free
|
||||
# The WRITE ACK, not the READ watchdog, releases the deferred blocks.
|
||||
if delay_free:
|
||||
assert sched._deferred_send_deadlines["req"][1] == "tx"
|
||||
else:
|
||||
assert "req" not in sched._deferred_send_deadlines
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_build_meta_saves_current_state_on_the_final_chunk():
|
||||
"""Chunked prefill must not ship block ids cached at admission: align-mode
|
||||
state slots move between chunks without appending a block."""
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
sched.is_producer = True
|
||||
set_role(moriio_common.ROLE.PRODUCER)
|
||||
params = {
|
||||
"do_remote_decode": True,
|
||||
"transfer_id": "tx",
|
||||
"remote_engine_id": "decode",
|
||||
"remote_block_ids": [],
|
||||
"remote_host": "127.0.0.1",
|
||||
"remote_handshake_port": 6000,
|
||||
"remote_notify_port": 6001,
|
||||
}
|
||||
req = _mk_request(list(range(9)), params=params)
|
||||
req.request_id = "req"
|
||||
req.num_computed_tokens = 0
|
||||
sched._reqs_need_save["req"] = (req, [[10, 11, 12], [1]])
|
||||
sched._req_kv_params["req"] = dict(params)
|
||||
|
||||
def _output(cached_req_ids, new_block_ids, num_scheduled):
|
||||
return SimpleNamespace(
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=cached_req_ids, new_block_ids=new_block_ids
|
||||
),
|
||||
num_scheduled_tokens={"req": num_scheduled},
|
||||
new_block_ids_to_zero=None,
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
req_ids={"req"},
|
||||
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
|
||||
boundary_state_offloads={},
|
||||
),
|
||||
)
|
||||
|
||||
# First chunk: not final, so nothing is saved yet.
|
||||
meta = sched.build_connector_meta(_output([], [], 4))
|
||||
assert not meta.reqs_to_save
|
||||
assert "req" in sched._reqs_need_pending_save
|
||||
|
||||
# Final chunk: stale cached ids plus new ids are ignored in favour of
|
||||
# the scheduler's current table, clamped to the prompt.
|
||||
req.num_computed_tokens = 4
|
||||
meta = sched.build_connector_meta(_output(["req"], [([77], [78])], 5))
|
||||
assert meta.reqs_to_save["req"].local_block_ids == [[51], [31, 32]]
|
||||
assert "req" not in sched._reqs_need_pending_save
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode,deadlines,expected",
|
||||
[
|
||||
(MoRIIOMode.WRITE, {"req": (0.0, "tx")}, True),
|
||||
(MoRIIOMode.WRITE, {}, False),
|
||||
(MoRIIOMode.READ, {"req": (0.0, "tx")}, False),
|
||||
],
|
||||
)
|
||||
def test_has_pending_push_work_tracks_write_deferred_frees(mode, deadlines, expected):
|
||||
connector = _FakeConnector(_FakeScheduler(_deferred_send_deadlines=deadlines))
|
||||
connector.mode = mode
|
||||
assert connector.has_pending_push_work() is expected
|
||||
|
||||
|
||||
def test_has_pending_push_work_is_false_on_worker_role():
|
||||
connector = _FakeConnector(None)
|
||||
connector.mode = MoRIIOMode.WRITE
|
||||
assert connector.has_pending_push_work() is False
|
||||
|
||||
|
||||
def test_loaded_kv_cache_groups_are_every_transfer_group():
|
||||
# The connector restores the request's own blocks, not a hash-addressed
|
||||
# prefix, so a transfer-enabled Mamba group counts as loaded too.
|
||||
connector = _FakeConnector(None)
|
||||
connector._kv_cache_config = SimpleNamespace(
|
||||
transfer_group_ids=(0, 2), prefix_cacheable_group_ids=(0,)
|
||||
)
|
||||
assert connector.get_loaded_kv_cache_group_ids(SimpleNamespace()) == (0, 2)
|
||||
|
||||
|
||||
def test_split_block_groups_rejects_non_hybrid_use():
|
||||
@@ -728,11 +950,15 @@ def test_session_build_rejects_per_layer_region_count_mismatch():
|
||||
worker._get_built_session("prefill")
|
||||
|
||||
|
||||
def test_register_kv_caches_rejects_hybrid_write_before_registration():
|
||||
worker = _FakeWorker(mode=MoRIIOMode.WRITE, _transfer_layer_names={"kda.0"})
|
||||
def test_register_kv_caches_rejects_invalid_hybrid_write_state_layout():
|
||||
worker = _FakeWorker(
|
||||
mode=MoRIIOMode.WRITE,
|
||||
_transfer_layer_names={"kda.0"},
|
||||
layer_to_spec={"kda.0": _mamba_spec(1)},
|
||||
)
|
||||
worker._is_mamba_layer = lambda _layer_name: True
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"):
|
||||
worker.register_kv_caches({"kda.0": object()})
|
||||
|
||||
|
||||
@@ -933,6 +1159,40 @@ def test_read_matches_only_requests_with_pending_remote_prefill(params, expected
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"has_mamba,expected", [(False, (32, True)), (True, (31, True))]
|
||||
)
|
||||
def test_write_consumer_matches_full_prompt_unless_hybrid(has_mamba, expected):
|
||||
"""Without a mamba group (attention-only, or HMA disabled) prefill pushed
|
||||
KV for all N tokens; with one it pushed h(N-1)."""
|
||||
scheduler = _FakeScheduler(
|
||||
is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=has_mamba
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
num_prompt_tokens=32, kv_transfer_params={"do_remote_prefill": True}
|
||||
)
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"is_producer,has_mamba,expected_prompt_len",
|
||||
[(True, True, 4), (True, False, 5), (False, True, 5)],
|
||||
)
|
||||
def test_on_new_request_truncates_only_hybrid_producers(
|
||||
is_producer, has_mamba, expected_prompt_len
|
||||
):
|
||||
"""Attention-only prefill (including models run with HMA disabled)
|
||||
computes all N prompt tokens; only a hybrid producer drops the last."""
|
||||
scheduler = _FakeScheduler(is_producer=is_producer, _has_mamba=has_mamba)
|
||||
request = _mk_request([10, 11, 12, 13, 14], params={"do_remote_decode": True})
|
||||
|
||||
scheduler.on_new_request(request)
|
||||
|
||||
assert request.num_prompt_tokens == expected_prompt_len
|
||||
assert len(request.prompt_token_ids) == expected_prompt_len
|
||||
|
||||
|
||||
def test_read_rematch_after_alloc_keeps_queued_transfer():
|
||||
"""Clearing the flag blocks another external hit, not the queued READ."""
|
||||
sched = _make_read_scheduler()
|
||||
@@ -981,12 +1241,37 @@ def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
|
||||
assert is_async is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [None, {}, {"do_remote_prefill": False}])
|
||||
def test_write_does_not_wait_again_after_receive_or_for_local_requests(params):
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
|
||||
req = _mk_request(list(range(10)))
|
||||
req.kv_transfer_params = params
|
||||
assert sched.get_num_new_matched_tokens(req, 0) == (0, False)
|
||||
|
||||
|
||||
def test_write_never_declares_an_empty_async_load():
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False)
|
||||
req = _mk_request(list(range(10)), params={"do_remote_prefill": True})
|
||||
assert sched.get_num_new_matched_tokens(req, 10) == (0, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt_len,computed,expected", [(16, 0, 15), (17, 16, 0), (1, 0, 0)]
|
||||
)
|
||||
def test_hybrid_write_advertises_only_the_state_computed_by_prefill(
|
||||
prompt_len, computed, expected
|
||||
):
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
|
||||
req = _mk_request(list(range(prompt_len)), params={"do_remote_prefill": True})
|
||||
assert sched.get_num_new_matched_tokens(req, computed) == (expected, expected > 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "num_computed_tokens", "expected", "is_async"),
|
||||
[
|
||||
(MoRIIOMode.READ, 0, 9, False),
|
||||
(MoRIIOMode.READ, 10, 0, False),
|
||||
(MoRIIOMode.WRITE, 2, 8, True),
|
||||
(MoRIIOMode.WRITE, 2, 7, True),
|
||||
],
|
||||
)
|
||||
def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
|
||||
|
||||
@@ -7,6 +7,7 @@ import threading
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -106,7 +107,7 @@ def _write_task(layer_name: str, transfer_id: str = "xfer") -> Any:
|
||||
request_id="req",
|
||||
transfer_id=transfer_id,
|
||||
dst_engine_id="remote-engine",
|
||||
local_block_ids=[1, 3],
|
||||
local_block_ids=[[1, 3]],
|
||||
remote_block_ids_hint=None,
|
||||
layer_name=layer_name,
|
||||
event=None,
|
||||
@@ -345,6 +346,8 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeWorker:
|
||||
_has_mamba: bool
|
||||
layer_to_group: dict[str, int]
|
||||
kv_caches: dict[str, torch.Tensor]
|
||||
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
|
||||
|
||||
@@ -364,11 +367,13 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
|
||||
return ([call_id], [call_id + 10], [call_id + 20])
|
||||
|
||||
fake_worker = FakeWorker()
|
||||
fake_worker._has_mamba = False
|
||||
fake_worker.layer_to_group = dict.fromkeys(kv_caches, 0)
|
||||
fake_worker.kv_caches = kv_caches
|
||||
fake_worker.layer_name_to_local_kv_cache_metadata = {name: [] for name in kv_caches}
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5])
|
||||
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
remote_meta = _remote_meta()
|
||||
|
||||
dense0_plan = writer._prepare_transfer_plan(
|
||||
@@ -395,7 +400,7 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
|
||||
|
||||
|
||||
def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5])
|
||||
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
@@ -411,6 +416,80 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
|
||||
assert writer._sealed_writes["xfer"] == 2
|
||||
|
||||
|
||||
def test_hybrid_write_attention_uses_its_own_group_not_the_first_group():
|
||||
worker = SimpleNamespace(
|
||||
_has_mamba=True,
|
||||
kv_caches={"draft": torch.empty((8, 1, 4, 3))},
|
||||
layer_to_group={"draft": 2},
|
||||
_region_session_indices=lambda _: [5],
|
||||
_compute_block_transfer_offsets=Mock(return_value=([1], [2], [3])),
|
||||
)
|
||||
writer = make_moriio_writer(worker)
|
||||
task = _write_task("draft")
|
||||
task.local_block_ids = [[10], [20], [30, 31]]
|
||||
remote = RemoteAllocInfo(block_ids=[[40], [50], [60, 61, 62]])
|
||||
meta = _remote_meta()
|
||||
plan = writer._prepare_transfer_plan(task, remote, meta)
|
||||
worker._compute_block_transfer_offsets.assert_called_once_with(
|
||||
"draft", [30, 31], [60, 61], meta, remote_engine_id=task.dst_engine_id
|
||||
)
|
||||
assert plan.sess_idx == 5
|
||||
|
||||
|
||||
def test_hybrid_write_posts_both_regions_before_layer_completion():
|
||||
from unittest.mock import Mock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorWorker,
|
||||
)
|
||||
|
||||
wrapper = _wrapper_for_messages()
|
||||
info = RemoteAllocInfo(block_ids=[[], [7], [9]])
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = info
|
||||
wrapper.write_remote_data = Mock(side_effect=["conv-status", "ssm-status"])
|
||||
worker = SimpleNamespace(
|
||||
moriio_wrapper=wrapper,
|
||||
tp_rank=0,
|
||||
get_engine_name_with_dp=lambda name, rank: name,
|
||||
_get_built_session=lambda name: (["conv-session", "ssm-session"], None),
|
||||
_is_mamba_layer=lambda name: True,
|
||||
_mamba_payload_index_by_layer={"kda": 2},
|
||||
_region_session_indices=lambda name: [0, 1],
|
||||
_compute_mamba_transfer_offsets=Mock(
|
||||
return_value=([10, 20, 30, 40], [50, 60, 70, 80], [2, 4, 8, 16], 3)
|
||||
),
|
||||
)
|
||||
worker._mamba_blocks_for_layer = lambda name, blocks: (
|
||||
MoRIIOConnectorWorker._mamba_blocks_for_layer(worker, name, blocks)
|
||||
)
|
||||
writer = make_moriio_writer(worker)
|
||||
task = _write_task("kda")
|
||||
task.local_block_ids = [[], [1], [3]]
|
||||
task.remote_tp_size = 2
|
||||
task.event = Mock()
|
||||
writer._execute_write_task(task)
|
||||
|
||||
task.event.synchronize.assert_called_once()
|
||||
worker._compute_mamba_transfer_offsets.assert_called_once_with(
|
||||
"kda", [3], [9], remote_tp_size=2
|
||||
)
|
||||
assert wrapper.write_remote_data.call_args_list[0].args == (
|
||||
[2, 4, 8],
|
||||
[10, 20, 30],
|
||||
[50, 60, 70],
|
||||
"conv-session",
|
||||
)
|
||||
assert wrapper.write_remote_data.call_args_list[1].args == (
|
||||
[16],
|
||||
[40],
|
||||
[80],
|
||||
"ssm-session",
|
||||
)
|
||||
assert info.transfer_statuses == ["conv-status", "ssm-status"]
|
||||
assert info.writes_done == 1
|
||||
assert not info.completion_notified
|
||||
|
||||
|
||||
def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
@@ -445,7 +524,7 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
)
|
||||
|
||||
wrapper = FakeWrapper()
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
|
||||
request_info = RemoteAllocInfo(block_ids=[[4, 5]], writes_expected=2)
|
||||
request_info.transfer_statuses.extend(["status-a", "status-b"])
|
||||
request_info.completion_request_id = "req"
|
||||
# Final notify port is resolved in _execute_write_task; finalize reads it
|
||||
@@ -487,6 +566,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
|
||||
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
|
||||
wrapper.lock = threading.Lock()
|
||||
wrapper._transfer_timeout = 1
|
||||
wrapper.moriio_engine = object()
|
||||
global_status = FakeStatus()
|
||||
scoped_status = FakeStatus()
|
||||
wrapper.transfer_status = [global_status]
|
||||
@@ -500,7 +580,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
|
||||
|
||||
def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 2
|
||||
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
|
||||
@@ -518,7 +598,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
|
||||
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 1
|
||||
writer._scheduled_layers["xfer"] = {"dense0"}
|
||||
@@ -610,7 +690,7 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
|
||||
|
||||
if expected == "remote_blocks":
|
||||
request_info = wrapper.done_remote_allocate_req_dict["xfer"]
|
||||
assert request_info.block_ids == [4, 5]
|
||||
assert request_info.block_ids == [[4, 5]]
|
||||
assert request_info.decode_dp_rank == 3
|
||||
elif expected == "write_done":
|
||||
assert wrapper.done_write_cache_req_ids == ["xfer"]
|
||||
|
||||
@@ -11,6 +11,8 @@ import importlib
|
||||
import importlib.util
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -127,3 +129,62 @@ def test_wait_blocks_until_terminal():
|
||||
assert time.monotonic() - start >= 0.15
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("timeout,expected_ms", [(0.05, 50), (0.0001, 1)])
|
||||
def test_native_wait_receives_all_statuses_and_a_positive_deadline(
|
||||
timeout, expected_ms
|
||||
):
|
||||
statuses = [_pending(), _ok()]
|
||||
engine = SimpleNamespace(
|
||||
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
|
||||
)
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=timeout)
|
||||
wrapper.waiting_for_transfer_complete(statuses)
|
||||
engine.wait_all.assert_called_once_with(statuses, timeout_ms=expected_ms)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"result,statuses,message",
|
||||
[
|
||||
("ERR_RDMA_OP", [_pending(), _bad()], "SQ full"),
|
||||
("IN_PROGRESS", [_pending(), _ok()], "timed out"),
|
||||
("ERR_INVALID_ARGS", [_pending()], "ERR_INVALID_ARGS"),
|
||||
],
|
||||
)
|
||||
def test_native_wait_reports_failure_or_timeout(result, statuses, message):
|
||||
engine = SimpleNamespace(
|
||||
wait_all=Mock(return_value=getattr(moriio_engine.StatusCode, result))
|
||||
)
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
with pytest.raises(TransferError, match=message):
|
||||
wrapper.waiting_for_transfer_complete(statuses)
|
||||
|
||||
|
||||
def test_wait_falls_back_without_native_status_enum(monkeypatch):
|
||||
monkeypatch.setattr(moriio_engine, "StatusCode", None)
|
||||
engine = SimpleNamespace(wait_all=Mock())
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
with pytest.raises(TransferError, match="SQ full"):
|
||||
wrapper.waiting_for_transfer_complete([_bad(), _ok()])
|
||||
engine.wait_all.assert_not_called()
|
||||
|
||||
|
||||
def test_nonblocking_poll_does_not_call_native_wait():
|
||||
engine = SimpleNamespace(wait_all=Mock())
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
assert wrapper.poll_transfer_batch([_pending()]) is TransferBatchState.PENDING
|
||||
wrapper.waiting_for_transfer_complete([])
|
||||
engine.wait_all.assert_not_called()
|
||||
|
||||
|
||||
def test_native_wait_drains_only_the_unscoped_status_snapshot():
|
||||
statuses = [_pending(), _ok()]
|
||||
engine = SimpleNamespace(
|
||||
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
|
||||
)
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
wrapper.transfer_status.extend(statuses)
|
||||
wrapper.waiting_for_transfer_complete()
|
||||
engine.wait_all.assert_called_once_with(statuses, timeout_ms=50)
|
||||
assert wrapper.transfer_status == []
|
||||
|
||||
@@ -14,6 +14,8 @@ import pytest
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ROLE,
|
||||
LayerTransferPlan,
|
||||
MoRIIOError,
|
||||
MoRIIOMode,
|
||||
RemoteAllocInfo,
|
||||
ReqMeta,
|
||||
@@ -96,7 +98,7 @@ def _write_task(
|
||||
request_id=request_id,
|
||||
transfer_id=transfer_id,
|
||||
dst_engine_id="remote-engine",
|
||||
local_block_ids=[1, 3],
|
||||
local_block_ids=[[1, 3]],
|
||||
remote_block_ids_hint=None,
|
||||
layer_name="layer0",
|
||||
event=None,
|
||||
@@ -249,6 +251,7 @@ def test_finalize_single_pod_port_uses_global_rank():
|
||||
|
||||
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = False
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
@@ -266,13 +269,14 @@ def test_write_blocks_for_req_forwards_routing_to_schedule_write():
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
|
||||
assert captured["remote_dp_size_local"] == 8
|
||||
assert captured["remote_ip"] == "10.0.0.1"
|
||||
assert captured["local_block_ids"] == [1]
|
||||
assert captured["remote_block_ids"] == [2]
|
||||
assert captured["local_block_ids"] == [[1]]
|
||||
assert captured["remote_block_ids"] == [[2]]
|
||||
assert not hasattr(worker, "multi_pod_hosts")
|
||||
|
||||
|
||||
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = False
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
@@ -290,8 +294,8 @@ def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
|
||||
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
|
||||
assert captured["remote_dp_size_local"] == 4
|
||||
assert captured["local_block_ids"] == [1]
|
||||
assert captured["remote_block_ids"] == [2]
|
||||
assert captured["local_block_ids"] == [[1]]
|
||||
assert captured["remote_block_ids"] == [[2]]
|
||||
|
||||
|
||||
def test_deferred_write_timeout_fails_transfer_on_every_decode_rank():
|
||||
@@ -373,3 +377,73 @@ def test_write_failed_is_reported_as_failed_recving_on_owning_rank():
|
||||
assert results.failed_recving == {"req-1"}
|
||||
assert wrapper.pop_finished_write_req_ids() == set()
|
||||
assert list(worker._unmatched_write_failures) == ["tOtherRank"]
|
||||
|
||||
|
||||
def test_hybrid_write_rejects_unequal_tp_before_scheduling_any_layer():
|
||||
from unittest.mock import Mock
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = True
|
||||
worker.world_size = 2
|
||||
worker.schedule_write_blocks = Mock()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
|
||||
worker._write_blocks_for_req("req", _req_meta(), "attention", kv_layer=None)
|
||||
|
||||
worker.schedule_write_blocks.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"notify,expected",
|
||||
[
|
||||
# Plain WRITE peers still send one flat attention list.
|
||||
([7, 8], [[7, 8]]),
|
||||
# Hybrid WRITE sends one list per transfer group, kept as-is even
|
||||
# when the first group is empty (full attention hit, state only).
|
||||
([[7], [9]], [[7], [9]]),
|
||||
([[], [9]], [[], [9]]),
|
||||
],
|
||||
)
|
||||
def test_remote_blocks_message_normalizes_to_grouped_block_ids(notify, expected):
|
||||
wrapper = _real_wrapper()
|
||||
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
|
||||
wrapper._handle_remote_blocks_message(
|
||||
{"transfer_id": "tA", "block_notify_list": notify, "decode_rank": 2}
|
||||
)
|
||||
info = wrapper.done_remote_allocate_req_dict["tA"]
|
||||
assert info.block_ids == expected
|
||||
assert info.decode_dp_rank == 2
|
||||
|
||||
|
||||
def test_empty_layer_plan_posts_no_rdma_write():
|
||||
from unittest.mock import Mock
|
||||
|
||||
wrapper = SimpleNamespace(write_remote_data=Mock(), write_remote_data_single=Mock())
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
plan = LayerTransferPlan(
|
||||
request_id="req",
|
||||
transfer_id="tA",
|
||||
layer_name="layer0",
|
||||
sess_idx=0,
|
||||
transfer_local_offsets=[],
|
||||
transfer_remote_offsets=[],
|
||||
transfer_sizes=[],
|
||||
)
|
||||
|
||||
assert writer._do_layer_write(plan, ["session"]) == []
|
||||
wrapper.write_remote_data.assert_not_called()
|
||||
wrapper.write_remote_data_single.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_state", [[], [1, 2]])
|
||||
def test_hybrid_write_requires_exactly_one_running_state_slot(local_state):
|
||||
worker = SimpleNamespace(
|
||||
_mamba_blocks_for_layer=lambda layer_name, block_ids: block_ids[1],
|
||||
)
|
||||
writer = make_moriio_writer(worker)
|
||||
task = _write_task("tA", "10.0.0.1")
|
||||
task.layer_name = "kda"
|
||||
task.local_block_ids = [[], local_state]
|
||||
|
||||
with pytest.raises(MoRIIOError, match="one running-state slot"):
|
||||
writer._prepare_mamba_transfer_plans(task, RemoteAllocInfo(block_ids=[[], [9]]))
|
||||
|
||||
@@ -805,6 +805,23 @@ class TestMultiConnectorStats:
|
||||
|
||||
child_metrics.observe.assert_called_once_with({"transfer_duration": [1.5]}, 2)
|
||||
|
||||
@pytest.mark.parametrize("with_exporter", [False, True])
|
||||
def test_prom_metrics_skips_stats_only_connectors(self, with_exporter):
|
||||
"""Stats-only children must not prevent supported metrics from exporting."""
|
||||
child_metrics = MagicMock()
|
||||
metrics = object.__new__(MultiKVConnectorPromMetrics)
|
||||
metrics._prom_metrics = (
|
||||
{"MooncakeStoreConnector": child_metrics} if with_exporter else {}
|
||||
)
|
||||
payload = {"MooncakeConnector": {"bytes_transferred": [1024]}}
|
||||
if with_exporter:
|
||||
payload["MooncakeStoreConnector"] = {"load_get_bytes": [2048]}
|
||||
|
||||
metrics.observe(payload, engine_idx=2)
|
||||
|
||||
if with_exporter:
|
||||
child_metrics.observe.assert_called_once_with({"load_get_bytes": [2048]}, 2)
|
||||
|
||||
def test_aggregate_same_connector(self):
|
||||
"""Test aggregating stats from the same connector type."""
|
||||
stats1 = MultiKVConnectorStats(
|
||||
|
||||
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p=None)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topp_only(self, batch_size: int, vocab_size: int):
|
||||
"""Test top-p only (k=None)."""
|
||||
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k=None, p=p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
|
||||
"""Test combined top-k and top-p."""
|
||||
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48, 64])
|
||||
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
|
||||
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
|
||||
"""Mixed batches at vocab sizes where the split top-p pipeline uses
|
||||
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
|
||||
k[1::4] = vocab_size
|
||||
p[2::4] = 1.0
|
||||
k[3::4] = vocab_size
|
||||
p[3::4] = 1.0
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
def test_both_disabled(self):
|
||||
"""Test when both k and p are None (should be no-op)."""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
|
||||
f"{kept_topk_topp.tolist()}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48])
|
||||
def test_skip_split_without_topp_only_rows(
|
||||
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""has_topp_only_rows=False skips the split pipeline; the result must
|
||||
not change when every top-p row also has a top-k, grammar rows
|
||||
(k >= #finite logits) included."""
|
||||
import vllm.v1.sample.ops.topk_topp_triton as tt
|
||||
|
||||
vocab_size = 32000
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
logits[1] = float("-inf")
|
||||
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
|
||||
logits[1, idx] = torch.randn(30, generator=self.generator)
|
||||
k = torch.randint(
|
||||
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
|
||||
)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
k[::4] = vocab_size
|
||||
p[::4] = 1.0
|
||||
|
||||
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
|
||||
monkeypatch.setattr(
|
||||
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
|
||||
)
|
||||
actual = tt.apply_top_k_top_p_triton(
|
||||
logits.clone(), k, p, has_topp_only_rows=False
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
def test_topp_tie_break_deterministic(self):
|
||||
"""Boundary-duplicate trimming must be deterministic across calls.
|
||||
|
||||
@@ -821,6 +871,36 @@ class TestTritonTopkTopp:
|
||||
)
|
||||
assert len(kept_idx) == pytorch_kept
|
||||
|
||||
def test_topp_ties_across_blocks(self):
|
||||
"""The split top-p tie budget must carry across a slice's blocks.
|
||||
|
||||
Regression: the split top-p mask kernel restarted its count of
|
||||
boundary-value copies at every 8192-entry block, so each block of a
|
||||
slice could keep up to `numkeep` copies of its own.
|
||||
"""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
|
||||
batch_size, vocab_size, num_ties = 64, 151936, 4096
|
||||
# 4096 tied tokens spread over many blocks, between 64 tokens above
|
||||
# and a low tail; the p=0.75 boundary lands mid-copy in the tie group.
|
||||
tie_idx = torch.arange(num_ties) * 37
|
||||
logits = torch.full((batch_size, vocab_size), -8.0)
|
||||
logits[:, tie_idx] = 0.0
|
||||
logits[:, -64:] = 4.0
|
||||
p = torch.full((batch_size,), 0.75, dtype=torch.float32)
|
||||
|
||||
kept = apply_top_k_top_p_triton(logits.clone(), None, p) != float("-inf")
|
||||
expected = apply_top_k_top_p_pytorch(logits.clone(), None, p) != float("-inf")
|
||||
num_kept, num_expected = kept.sum(dim=-1), expected.sum(dim=-1)
|
||||
assert torch.equal(num_kept, num_expected), (
|
||||
f"kept {num_kept.unique().tolist()}, "
|
||||
f"expected {num_expected.unique().tolist()}"
|
||||
)
|
||||
# Kept copies of the boundary value are the first ones in index order.
|
||||
kept_ties = kept[:, tie_idx]
|
||||
num_kept_ties = kept_ties.sum(dim=-1, keepdim=True)
|
||||
assert torch.equal(kept_ties, torch.arange(num_ties) < num_kept_ties)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# FlashInfer top-k/top-p robustness tests
|
||||
|
||||
@@ -245,6 +245,55 @@ def test_connector_reports_configured_cache_source(
|
||||
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_layers", [1, 2])
|
||||
@pytest.mark.parametrize("backend", ["cpu", "disk"])
|
||||
def test_connector_explicit_cpu_blocks_align_uneven_stages(
|
||||
tmp_path, num_layers: int, backend: str
|
||||
):
|
||||
"""The same configured pool size must fit lighter and heavier PP stages."""
|
||||
config = _make_kv_cache_config(num_blocks=16)
|
||||
names = [f"layer_{i}" for i in range(num_layers)]
|
||||
config.kv_cache_groups[0].layer_names = names
|
||||
config.kv_cache_tensors[0].layers = names
|
||||
config.kv_cache_tensors[0].size *= num_layers
|
||||
vllm_config = _make_vllm_config()
|
||||
extra = {
|
||||
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 8,
|
||||
"num_cpu_blocks": 4,
|
||||
"kv_offload_backend": backend,
|
||||
}
|
||||
if backend == "disk":
|
||||
extra.update(
|
||||
disk_path=str(tmp_path / "offload.bin"),
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = extra
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, config
|
||||
)
|
||||
assert connector.scheduler_manager is not None
|
||||
assert connector.scheduler_manager.num_cpu_blocks == 4
|
||||
assert len(connector.scheduler_manager.cpu_block_pool.blocks) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count, capacity_bytes",
|
||||
[(count, _BYTES_PER_BLOCK * 8) for count in [0, -1, True, 1.5, "4", 9]]
|
||||
+ [(1, _BYTES_PER_BLOCK // 2)],
|
||||
)
|
||||
def test_connector_rejects_invalid_explicit_cpu_blocks(count, capacity_bytes):
|
||||
config = _make_kv_cache_config(num_blocks=16)
|
||||
vllm_config = _make_vllm_config()
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = {
|
||||
"cpu_bytes_to_use_per_rank": capacity_bytes,
|
||||
"num_cpu_blocks": count,
|
||||
}
|
||||
with pytest.raises(ValueError, match="num_cpu_blocks"):
|
||||
SimpleCPUOffloadConnector(vllm_config, KVConnectorRole.SCHEDULER, config)
|
||||
|
||||
|
||||
def make_scheduler(
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
|
||||
@@ -371,6 +371,69 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_layers", [1, 2])
|
||||
def test_explicit_cpu_blocks_bound_live_worker_allocation(monkeypatch, num_layers):
|
||||
"""Different live KV strides must honor the same explicit CPU pool size."""
|
||||
num_blocks = 4
|
||||
names = [f"layer.{i}" for i in range(num_layers)]
|
||||
raw = torch.zeros(
|
||||
num_layers * num_blocks * BLOCK_BYTES, dtype=torch.int8, device="cuda"
|
||||
)
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=raw.nbytes,
|
||||
layers=names,
|
||||
layer_stride=num_blocks * BLOCK_BYTES,
|
||||
block_stride=BLOCK_BYTES,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
)
|
||||
caches = dict(zip(names, raw.view(num_layers, num_blocks, BLOCK_BYTES)))
|
||||
worker = SimpleCPUOffloadWorker(None, config, 8 * BLOCK_BYTES, num_cpu_blocks=4)
|
||||
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
|
||||
try:
|
||||
worker.register_kv_caches(caches)
|
||||
assert worker.num_cpu_blocks == 4
|
||||
assert worker.cpu_kv_caches is not None
|
||||
assert all(t.shape[0] == 4 for t in worker.cpu_kv_caches.values())
|
||||
assert sum(t.nbytes for t in worker.cpu_kv_caches.values()) <= 8 * BLOCK_BYTES
|
||||
finally:
|
||||
if worker._backend is not None:
|
||||
worker._backend.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"capacity_bytes, count, local_capacity",
|
||||
[(8 * BLOCK_BYTES, 5, 4), (BLOCK_BYTES, 1, 0)],
|
||||
)
|
||||
def test_explicit_cpu_blocks_reject_heavier_worker_before_allocation(
|
||||
capacity_bytes, count, local_capacity
|
||||
):
|
||||
"""A count that fits the scheduler's stage must fail on a smaller worker."""
|
||||
names = ["layer.0", "layer.1"]
|
||||
raw = torch.zeros(2 * 4 * BLOCK_BYTES, dtype=torch.int8, device="cuda")
|
||||
config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=raw.nbytes,
|
||||
layers=names,
|
||||
layer_stride=4 * BLOCK_BYTES,
|
||||
block_stride=BLOCK_BYTES,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
)
|
||||
worker = SimpleCPUOffloadWorker(None, config, capacity_bytes, num_cpu_blocks=count)
|
||||
caches = dict(zip(names, raw.view(2, 4, BLOCK_BYTES)))
|
||||
with pytest.raises(ValueError, match=f"local offload capacity {local_capacity}"):
|
||||
worker.register_kv_caches(caches)
|
||||
assert worker.cpu_kv_caches is None
|
||||
|
||||
|
||||
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
|
||||
num_blocks = 4
|
||||
block_bytes = 32
|
||||
|
||||
@@ -1157,6 +1157,91 @@ def test_chunked_requests_match_full_batch(has_draft_logits: bool):
|
||||
assert torch.equal(chunked_sampled[valid], sampled[valid])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_block_verification", [False, True])
|
||||
@pytest.mark.parametrize("has_draft_logits", [False, True])
|
||||
def test_adaptive_draft_lengths_match_and_bonus_oracle(
|
||||
use_block_verification: bool, has_draft_logits: bool
|
||||
):
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
K = 3
|
||||
num_drafts = [3, 1, 0, 2, 3, 2]
|
||||
temperatures = [1.0, 0.0, 0.0, 0.6, 0.0, 1.0]
|
||||
greedy_num_matches = {1: 1, 2: 0, 4: 2}
|
||||
num_reqs = len(num_drafts)
|
||||
|
||||
num_logits_per_req = torch.tensor(num_drafts, device=device) + 1
|
||||
cu_num_logits = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
cu_num_logits[1:] = num_logits_per_req.cumsum(0)
|
||||
num_logits = int(cu_num_logits[-1])
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
expanded_idx_mapping = idx_mapping.repeat_interleave(num_logits_per_req)
|
||||
expanded_local_pos = (
|
||||
torch.arange(num_logits, dtype=torch.int32, device=device)
|
||||
- cu_num_logits[expanded_idx_mapping]
|
||||
)
|
||||
|
||||
draft_logits = torch.randn(num_reqs, K, VOCAB_SIZE, device=device)
|
||||
target_logits = torch.randn(num_logits, VOCAB_SIZE, device=device)
|
||||
# Allocate at exactly num_logits so a lookahead past the batch is OOB.
|
||||
draft_sampled = torch.randint(VOCAB_SIZE, (num_logits,), device=device)
|
||||
target_argmax = target_logits.argmax(dim=-1)
|
||||
for req_idx, num_matches in greedy_num_matches.items():
|
||||
lo = int(cu_num_logits[req_idx])
|
||||
for step in range(num_drafts[req_idx]):
|
||||
token = target_argmax[lo + step]
|
||||
if step >= num_matches:
|
||||
token = (token + 1) % VOCAB_SIZE
|
||||
draft_sampled[lo + step + 1] = token
|
||||
|
||||
inputs = dict(
|
||||
target_logits=target_logits,
|
||||
draft_logits=draft_logits if has_draft_logits else None,
|
||||
draft_sampled=draft_sampled,
|
||||
cu_num_logits=cu_num_logits,
|
||||
pos=torch.arange(num_logits, dtype=torch.int32, device=device),
|
||||
idx_mapping=idx_mapping,
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
expanded_local_pos=expanded_local_pos,
|
||||
temperature=torch.tensor(temperatures, device=device),
|
||||
seed=torch.arange(num_reqs, dtype=torch.int64, device=device),
|
||||
num_speculative_steps=K,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
sampled, num_sampled = rejection_sample(**inputs)
|
||||
|
||||
for req_idx, num_matches in greedy_num_matches.items():
|
||||
lo = int(cu_num_logits[req_idx])
|
||||
n = num_matches + 1
|
||||
assert int(num_sampled[req_idx]) == n, f"greedy request {req_idx}"
|
||||
assert torch.equal(sampled[req_idx, :n], target_argmax[lo : lo + n]), (
|
||||
f"greedy request {req_idx}"
|
||||
)
|
||||
|
||||
for req_idx in range(num_reqs):
|
||||
lo, hi = int(cu_num_logits[req_idx]), int(cu_num_logits[req_idx + 1])
|
||||
req_inputs = dict(inputs)
|
||||
for name in (
|
||||
"target_logits",
|
||||
"draft_sampled",
|
||||
"pos",
|
||||
"expanded_idx_mapping",
|
||||
"expanded_local_pos",
|
||||
):
|
||||
req_inputs[name] = inputs[name][lo:hi]
|
||||
req_inputs["cu_num_logits"] = cu_num_logits[req_idx : req_idx + 2] - lo
|
||||
req_inputs["idx_mapping"] = idx_mapping[req_idx : req_idx + 1]
|
||||
|
||||
req_sampled, req_num_sampled = rejection_sample(**req_inputs)
|
||||
|
||||
n = int(req_num_sampled[0])
|
||||
assert n <= num_drafts[req_idx] + 1
|
||||
assert int(num_sampled[req_idx]) == n, f"request {req_idx} mismatch"
|
||||
assert torch.equal(sampled[req_idx, :n], req_sampled[0, :n]), (
|
||||
f"request {req_idx} mismatch"
|
||||
)
|
||||
|
||||
|
||||
def _residual_recovery_logits(
|
||||
inputs: dict,
|
||||
num_sampled: torch.Tensor,
|
||||
|
||||
@@ -109,6 +109,30 @@ def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
|
||||
assert sampler.needs_logits_processing[3] == admitted
|
||||
|
||||
|
||||
def test_has_topp_only_rows():
|
||||
"""A row is top-p-only when it has top-p and no effective top-k; top_k
|
||||
of 0 (the default) or at least the logits width disables top-k."""
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, SamplingParams(top_k=10, top_p=0.9))
|
||||
sampler.add_request(1, SamplingParams(top_k=10))
|
||||
sampler.add_request(2, SamplingParams(top_p=0.9))
|
||||
sampler.add_request(3, SamplingParams(top_k=VOCAB_SIZE + 1, top_p=0.9))
|
||||
states = sampler.sampling_states
|
||||
states.apply_staged_writes()
|
||||
|
||||
def has_topp_only_rows(idx: list[int], vocab_size: int = VOCAB_SIZE) -> bool:
|
||||
idx_np = np.array(idx, dtype=np.int32)
|
||||
idx_gpu = torch.from_numpy(idx_np).to(DEVICE)
|
||||
return states.get_top_k_top_p(idx_gpu, idx_np, vocab_size)[2]
|
||||
|
||||
assert not has_topp_only_rows([0, 1])
|
||||
assert has_topp_only_rows([0, 2])
|
||||
assert has_topp_only_rows([3])
|
||||
# The kernel compares top_k against the logits width, so a top_k at or
|
||||
# above a narrower logits width counts as no top-k.
|
||||
assert has_topp_only_rows([0], vocab_size=10)
|
||||
|
||||
|
||||
def test_logits_processing_cache_only_checks_active_requests():
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, SamplingParams(temperature=0.0))
|
||||
|
||||
Reference in New Issue
Block a user