Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -5,8 +5,7 @@
|
||||
The decode path hands the kernel column slices of the merged ``q|k|v`` conv
|
||||
output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are
|
||||
token-strided rather than contiguous. Both backends must match a pure-PyTorch
|
||||
recurrence. CUDA reads these slices in place and rejects unsupported layouts;
|
||||
the ROCm wrapper makes contiguous copies before launching its kernel.
|
||||
recurrence, read these slices in place and reject unsupported layouts.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -166,8 +165,8 @@ def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Only the CUDA implementation requires zero-copy token-strided layouts",
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Both implementations require zero-copy token-strided layouts",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_recurrent_kda_rejects_unaddressable_layouts():
|
||||
@@ -193,3 +192,170 @@ def test_fused_recurrent_kda_rejects_unaddressable_layouts():
|
||||
broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"]
|
||||
with pytest.raises(AssertionError, match=r"torch.Size"):
|
||||
run_kernel(broken, state)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset", [16, 144, 256])
|
||||
@pytest.mark.parametrize("dim_first", [False, True])
|
||||
@pytest.mark.parametrize("num_spec", [0, 3])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset):
|
||||
"""Restoring both cached states must reproduce an uninterrupted prefill."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.kda_checkpoint import (
|
||||
FlashKDAPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.models.glm5next.common import kda
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata
|
||||
from vllm.v1.worker.workspace import WorkspaceManager
|
||||
|
||||
if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda":
|
||||
pytest.skip("requires FlashKDA")
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
length, heads = 273, 2
|
||||
width, state_len = 3 * heads * D, 3 + num_spec
|
||||
layer = object.__new__(kda.Glm5NextLinearAttention)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.prefix = "layer.0"
|
||||
layer.local_num_heads = heads
|
||||
layer.head_dim = D
|
||||
layer.conv_size = 4
|
||||
layer.local_projection_size = heads * D
|
||||
layer.kda_safe_gate = True
|
||||
layer.kda_lower_bound = -5.0
|
||||
layer.kda_prefill_backend = "flashkda"
|
||||
layer._conv_state_dim_first = dim_first
|
||||
layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1
|
||||
layer.q_conv1d = SimpleNamespace(bias=None)
|
||||
layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1
|
||||
layer.dt_bias = torch.randn(heads * D, device=device) * 0.1
|
||||
state_shape = (2, heads, D, D)
|
||||
layer._flashkda_buffer_specs = (
|
||||
(state_shape, torch.float32),
|
||||
(state_shape, torch.float32),
|
||||
(
|
||||
(torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),),
|
||||
torch.uint8,
|
||||
),
|
||||
((1, 2 * length, heads, D), torch.bfloat16),
|
||||
)
|
||||
layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3)
|
||||
workspace = WorkspaceManager(device)
|
||||
monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace)
|
||||
conv_shape = (width, state_len) if dim_first else (state_len, width)
|
||||
num_slots = 5 + (num_spec + 1 if num_spec else 0)
|
||||
conv_storage = torch.zeros(
|
||||
num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape)
|
||||
recurrent_storage = torch.zeros(
|
||||
num_slots, heads * D * D + 8, device=device, dtype=torch.float32
|
||||
)
|
||||
recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D)
|
||||
layer.kv_cache = (conv, recurrent)
|
||||
projected = torch.randn(
|
||||
2 * length, width + heads, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0)
|
||||
raw_before = raw_qkv.clone()
|
||||
gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16)
|
||||
|
||||
def run(qkv, g, b, slots, has_initial, checkpoint=None):
|
||||
n = qkv.shape[0] // 2
|
||||
cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32)
|
||||
nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata(
|
||||
cu_cpu, device=device
|
||||
)
|
||||
metadata = GDNAttentionMetadata(
|
||||
num_prefills=2,
|
||||
num_prefill_tokens=2 * n,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_spec_decodes=0,
|
||||
num_spec_decode_tokens=0,
|
||||
num_actual_tokens=2 * n,
|
||||
has_initial_state=torch.tensor(has_initial, device=device),
|
||||
non_spec_query_start_loc=cu_cpu.to(device),
|
||||
non_spec_state_indices_tensor=torch.tensor(
|
||||
slots, dtype=torch.int32, device=device
|
||||
),
|
||||
checkpoint=checkpoint,
|
||||
nums_dict=nums,
|
||||
batch_ptr=batch_ptr,
|
||||
token_chunk_offset_ptr=chunk_offsets,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kda,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}),
|
||||
)
|
||||
spec_tokens = num_spec + 1 if num_spec else 0
|
||||
if spec_tokens:
|
||||
qkv = torch.cat([qkv[:spec_tokens], qkv])
|
||||
g = torch.cat([g[:, :spec_tokens], g], dim=1)
|
||||
b = torch.cat([b[:, :spec_tokens], b], dim=1)
|
||||
metadata.num_spec_decodes = 1
|
||||
metadata.num_spec_decode_tokens = spec_tokens
|
||||
metadata.num_actual_tokens += spec_tokens
|
||||
metadata.spec_sequence_masks = torch.tensor(
|
||||
[True, False, False], device=device
|
||||
)
|
||||
metadata.spec_query_start_loc = torch.tensor(
|
||||
[0, spec_tokens], dtype=torch.int32, device=device
|
||||
)
|
||||
metadata.spec_state_indices_tensor = torch.arange(
|
||||
5, num_slots, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
metadata.spec_token_indx = torch.arange(spec_tokens, device=device)
|
||||
metadata.non_spec_token_indx = torch.arange(
|
||||
spec_tokens, qkv.shape[0], device=device
|
||||
)
|
||||
metadata.num_accepted_tokens = torch.ones(
|
||||
1, dtype=torch.int32, device=device
|
||||
)
|
||||
out = torch.empty(
|
||||
1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
layer._forward(qkv, g, b, out)
|
||||
return out[:, spec_tokens:]
|
||||
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
torch.tensor([offset, 0], dtype=torch.int32, device=device),
|
||||
torch.tensor([3, 0], dtype=torch.int32, device=device),
|
||||
)
|
||||
expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint)
|
||||
expected_state = recurrent[1].clone()
|
||||
saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone()
|
||||
torch.testing.assert_close(raw_qkv, raw_before)
|
||||
conv_view = conv if dim_first else conv.transpose(-1, -2)
|
||||
torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T)
|
||||
assert torch.count_nonzero(conv_view[3, :, 3:]) == 0
|
||||
assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0
|
||||
|
||||
run(
|
||||
torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]),
|
||||
torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1),
|
||||
torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1),
|
||||
[1, 2],
|
||||
[False, False],
|
||||
)
|
||||
torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03)
|
||||
|
||||
suffix = slice(offset, length)
|
||||
qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]])
|
||||
g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1)
|
||||
b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1)
|
||||
conv[4].copy_(saved_conv)
|
||||
recurrent[4].copy_(saved_recurrent)
|
||||
actual = run(qkv, g, b, [3, 4], [True, True])
|
||||
torch.testing.assert_close(
|
||||
actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03
|
||||
)
|
||||
torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03)
|
||||
|
||||
@@ -15,11 +15,11 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.layers import linear
|
||||
from vllm.models.glm5next.common import model as glm_model
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig
|
||||
|
||||
HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
|
||||
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm KDA prefill checkpoint: where it is taken and what it stores.
|
||||
|
||||
Two things have to line up for a checkpoint to be safe: the metadata must name
|
||||
a position the chunk kernel will actually write, and the conv window saved
|
||||
there must be the one a resumed prefill would have left.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDAMetadataBuilder
|
||||
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
|
||||
KimiK3ROCmKDAPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
KDA_CHECKPOINT_ALIGNMENT,
|
||||
is_fused_kda_chunk_supported,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_rocm()
|
||||
and torch.cuda.is_available()
|
||||
and is_fused_kda_chunk_supported()
|
||||
),
|
||||
reason="The ROCm KDA prefill checkpoint is exported by the fused chunk "
|
||||
"kernel, which needs a gfx950 build that includes it",
|
||||
)
|
||||
|
||||
# A mamba block size the chunk walk lands on. The real K3 value is 768; any
|
||||
# multiple of the kernel's 64-token chunk behaves the same and keeps the
|
||||
# fixtures small.
|
||||
BLOCK_SIZE = 64
|
||||
|
||||
|
||||
class _StubSpeculativeConfig:
|
||||
method = "dspark"
|
||||
parallel_drafting = False
|
||||
|
||||
def __init__(self, num_speculative_tokens: int) -> None:
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
|
||||
def use_eagle_block_drop(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _builder(
|
||||
num_prefill_checkpoint_blocks: int = 1,
|
||||
mamba_cache_mode: str = "align",
|
||||
block_size: int = BLOCK_SIZE,
|
||||
prefix_match_unit: int | None = None,
|
||||
num_spec: int = 0,
|
||||
) -> KimiK3ROCmKDAMetadataBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B", block_size=block_size
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
|
||||
if num_spec:
|
||||
vllm_config.speculative_config = _StubSpeculativeConfig(num_spec)
|
||||
return KimiK3ROCmKDAMetadataBuilder(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=(
|
||||
KDA_CHECKPOINT_ALIGNMENT if num_prefill_checkpoint_blocks else None
|
||||
),
|
||||
num_speculative_blocks=num_spec,
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
|
||||
|
||||
def _build(batch: BatchSpec, block_size: int = BLOCK_SIZE, **kw):
|
||||
common = create_common_attn_metadata(
|
||||
batch, block_size, torch.device("cuda"), arange_block_indices=True
|
||||
)
|
||||
return _builder(block_size=block_size, **kw).build(0, common)
|
||||
|
||||
|
||||
def _i32(values: list[int]) -> torch.Tensor:
|
||||
return torch.tensor(values, dtype=torch.int32, device="cuda")
|
||||
|
||||
|
||||
def test_checkpoint_metadata_targets_last_aligned_boundary() -> None:
|
||||
# Slot p holds the state after (p + 1) * block_size tokens, so a 200-token
|
||||
# prompt checkpoints at 192 into slot 2 and a 128-token one at 64 into slot
|
||||
# 0. The block table is an arange, so request 1 owns blocks 4..7.
|
||||
md = _build(BatchSpec(seq_lens=[200, 128], query_lens=[200, 128]))
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 64]))
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([2, 4]))
|
||||
|
||||
|
||||
def test_checkpoint_offset_is_relative_to_query_start() -> None:
|
||||
# 256 tokens computed, 144 more this step: the boundary at 384 lands 128
|
||||
# tokens into the chunk, and cdiv(400, 64) - 2 = 5 is its slot.
|
||||
md = _build(BatchSpec(seq_lens=[400, 64], query_lens=[144, 64]))
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([128, 0]))
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([5, NULL_BLOCK_ID]))
|
||||
|
||||
|
||||
def test_checkpoint_metadata_avoids_live_state_row() -> None:
|
||||
# Writing the row the walk itself owns would race it.
|
||||
md = _build(BatchSpec(seq_lens=[200, 130], query_lens=[200, 130]))
|
||||
|
||||
assert md.checkpoint is not None
|
||||
# `mamba_get_block_table_tensor` hands the walk block (seq_len - 1) // bs.
|
||||
live = torch.tensor([(200 - 1) // BLOCK_SIZE, (130 - 1) // BLOCK_SIZE])
|
||||
table = torch.arange(2 * 4).view(2, 4)
|
||||
live_rows = table[torch.arange(2), live].cuda()
|
||||
assert not bool((md.checkpoint.state_indices == live_rows).any())
|
||||
|
||||
|
||||
def test_checkpoint_metadata_skips_chunk_without_boundary() -> None:
|
||||
# 128 computed, 40 more: the only boundary at 128 is the state this chunk
|
||||
# started from, so there is nothing new to snapshot.
|
||||
assert _build(BatchSpec(seq_lens=[168], query_lens=[40])).checkpoint is None
|
||||
|
||||
|
||||
def test_checkpoint_metadata_skips_null_reservation() -> None:
|
||||
# Only the chunk that finishes a prompt gets a block reserved; every
|
||||
# earlier one leaves the shared null block, which nothing hashes and
|
||||
# writing it would corrupt every other request's scratch row.
|
||||
batch = BatchSpec(seq_lens=[200], query_lens=[200])
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
|
||||
)
|
||||
# cdiv(200, 64) - 2 = 2 is the checkpoint slot.
|
||||
common.block_table_tensor[0, 2] = NULL_BLOCK_ID
|
||||
|
||||
md = _builder().build(0, common)
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([NULL_BLOCK_ID]))
|
||||
|
||||
|
||||
def test_checkpoint_metadata_absent_without_reserved_block() -> None:
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[200], query_lens=[200]),
|
||||
num_prefill_checkpoint_blocks=0,
|
||||
)
|
||||
assert getattr(md, "checkpoint", None) is None
|
||||
|
||||
|
||||
def test_spec_checkpoint_metadata_gathers_non_spec_rows_by_index() -> None:
|
||||
# gdn_attn selects the non-spec rows by mask and reclassifies their
|
||||
# one-token decodes as prefills, so the chunk kernel is handed rows 1, 3
|
||||
# and 4 here while a num_decodes: slice would hand it rows 0, 1 and 2.
|
||||
num_spec = 2
|
||||
batch = BatchSpec(
|
||||
seq_lens=[300, 200, 400, 512, 128],
|
||||
query_lens=[num_spec + 1, 200, num_spec + 1, 1, 128],
|
||||
)
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
|
||||
)
|
||||
# Align mode reads 1 + num_speculative_blocks columns, so widen the table.
|
||||
rows, cols = common.block_table_tensor.shape
|
||||
width = cols + num_spec
|
||||
common.block_table_tensor = torch.arange(
|
||||
rows * width, dtype=torch.int32, device="cuda"
|
||||
).view(rows, width)
|
||||
|
||||
md = _builder(num_spec=num_spec).build(
|
||||
0,
|
||||
common,
|
||||
num_accepted_tokens=torch.ones(rows, dtype=torch.int32, device="cuda"),
|
||||
num_decode_draft_tokens_cpu=torch.tensor(
|
||||
[num_spec, -1, num_spec, -1, -1], dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
assert md.spec_sequence_masks is not None
|
||||
assert md.num_decodes == 0 and md.num_prefills == 3
|
||||
assert md.checkpoint is not None
|
||||
# Row 3 resumes mid-block, so its chunk holds no reusable boundary.
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 0, 64]))
|
||||
# Request r owns blocks r*width.., so slot cdiv(200, 64) - 2 = 2 for
|
||||
# request 1 and cdiv(128, 64) - 2 = 0 for request 4.
|
||||
torch.testing.assert_close(
|
||||
md.checkpoint.state_indices, _i32([width + 2, NULL_BLOCK_ID, 4 * width])
|
||||
)
|
||||
|
||||
|
||||
def test_checkpoint_metadata_absent_outside_align_mode() -> None:
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[200], query_lens=[200]),
|
||||
mamba_cache_mode="none",
|
||||
)
|
||||
assert getattr(md, "checkpoint", None) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conv half
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DIM = 384
|
||||
WIDTH = 4
|
||||
STATE_LEN = WIDTH - 1
|
||||
|
||||
|
||||
def store_conv_checkpoints(x, conv_state, cu, offsets, rows, state_len) -> None:
|
||||
KimiK3ROCmKDAPrefillCheckpointExporter(state_len=state_len).export(
|
||||
MambaPrefillCheckpointMetadata(offsets, rows),
|
||||
raw_qkv=x,
|
||||
conv_state=conv_state,
|
||||
cu_seqlens=cu,
|
||||
)
|
||||
|
||||
|
||||
def _conv_fixture(
|
||||
seqlens: list[int], slots: int = 16, seed: int = 0, num_spec: int = 0
|
||||
):
|
||||
# kda_state_shape widens the conv row to conv_kernel_size - 1 + num_spec,
|
||||
# but the window every reader starts from stays the leading STATE_LEN.
|
||||
torch.manual_seed(seed)
|
||||
total = sum(seqlens)
|
||||
x = torch.randn(total, DIM, device="cuda", dtype=torch.bfloat16)
|
||||
cu = torch.tensor(
|
||||
[0] + torch.tensor(seqlens).cumsum(0).tolist(),
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
conv_state = torch.full(
|
||||
(slots, DIM, STATE_LEN + num_spec),
|
||||
float("nan"),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return x, cu, conv_state
|
||||
|
||||
|
||||
def test_conv_checkpoint_saves_window_before_offset() -> None:
|
||||
seqlens = [200, 130]
|
||||
x, cu, conv_state = _conv_fixture(seqlens)
|
||||
offsets = _i32([192, 128])
|
||||
rows = _i32([5, 9])
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, offsets, rows, STATE_LEN)
|
||||
|
||||
for seq, (offset, row) in enumerate(zip([192, 128], [5, 9])):
|
||||
end = int(cu[seq]) + offset
|
||||
expected = x[end - STATE_LEN : end].transpose(0, 1)
|
||||
torch.testing.assert_close(conv_state[row], expected)
|
||||
|
||||
|
||||
def test_conv_checkpoint_skips_opted_out_sequences() -> None:
|
||||
x, cu, conv_state = _conv_fixture([200, 200])
|
||||
before = conv_state.clone()
|
||||
|
||||
store_conv_checkpoints(
|
||||
x, conv_state, cu, _i32([0, 192]), _i32([5, NULL_BLOCK_ID]), STATE_LEN
|
||||
)
|
||||
|
||||
assert torch.equal(conv_state.isnan(), before.isnan())
|
||||
|
||||
|
||||
def test_exporter_stores_conv_window_through_shared_interface() -> None:
|
||||
x, cu, conv_state = _conv_fixture([200, 200])
|
||||
exporter = KimiK3ROCmKDAPrefillCheckpointExporter(state_len=STATE_LEN)
|
||||
assert isinstance(exporter, MambaPrefillCheckpointExporter)
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
_i32([192, 0]), _i32([5, NULL_BLOCK_ID])
|
||||
)
|
||||
|
||||
exporter.export(checkpoint, raw_qkv=x, conv_state=conv_state, cu_seqlens=cu)
|
||||
|
||||
torch.testing.assert_close(conv_state[5], x[192 - STATE_LEN : 192].transpose(0, 1))
|
||||
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
|
||||
untouched[5] = False
|
||||
assert bool(conv_state[untouched].isnan().all())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_spec", [0, 2, 8])
|
||||
def test_conv_checkpoint_matches_prefill_truncated_at_offset(num_spec: int) -> None:
|
||||
# causal_conv1d_fn's own state write is the reference: a prefill that
|
||||
# stopped at the offset leaves exactly the row a later prefix-cache hit
|
||||
# reads back. Speculative decoding widens the row without moving that
|
||||
# window, so a checkpoint sized off the row would sit num_spec tokens in
|
||||
# the past.
|
||||
seqlens = [200, 130]
|
||||
offsets = [192, 64]
|
||||
x, cu, conv_state = _conv_fixture(seqlens, seed=3, num_spec=num_spec)
|
||||
weight = torch.randn(DIM, WIDTH, device="cuda", dtype=torch.bfloat16) * 0.3
|
||||
bias = torch.randn(DIM, device="cuda", dtype=torch.bfloat16)
|
||||
rows = [5, 9]
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32(offsets), _i32(rows), STATE_LEN)
|
||||
|
||||
# A prefill of just the prefix, into its own cache. Row 0 is the null block
|
||||
# the conv kernel skips by design, so the reference rows start at 1.
|
||||
prefix = torch.cat(
|
||||
[x[int(cu[n]) : int(cu[n]) + off] for n, off in enumerate(offsets)]
|
||||
)
|
||||
prefix_cu = torch.tensor(
|
||||
[0] + torch.tensor(offsets).cumsum(0).tolist(),
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
reference = torch.zeros_like(conv_state)
|
||||
causal_conv1d_fn(
|
||||
prefix.transpose(0, 1),
|
||||
weight,
|
||||
bias,
|
||||
activation="silu",
|
||||
conv_states=reference,
|
||||
has_initial_state=torch.zeros(2, dtype=torch.bool, device="cuda"),
|
||||
cache_indices=_i32([1, 2]),
|
||||
query_start_loc=prefix_cu,
|
||||
)
|
||||
|
||||
# causal_conv1d_fn writes only the leading STATE_LEN columns.
|
||||
window = slice(None), slice(None, STATE_LEN)
|
||||
torch.testing.assert_close(conv_state[rows[0]][window], reference[1][window])
|
||||
torch.testing.assert_close(conv_state[rows[1]][window], reference[2][window])
|
||||
|
||||
|
||||
def test_conv_checkpoint_leaves_spec_tail_of_row_alone() -> None:
|
||||
# The draft-token columns belong to causal_conv1d_update. Filling the whole
|
||||
# row would hand every reader, which indexes from column 0, a window
|
||||
# shifted num_spec tokens into the past.
|
||||
num_spec = 4
|
||||
x, cu, conv_state = _conv_fixture([200], num_spec=num_spec, seed=11)
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([5]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(
|
||||
conv_state[5, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
|
||||
)
|
||||
assert bool(conv_state[5, :, STATE_LEN:].isnan().all()), (
|
||||
"the checkpoint wrote into the speculative-decode tail of the row"
|
||||
)
|
||||
|
||||
|
||||
def test_conv_checkpoint_honours_transposed_cache_view() -> None:
|
||||
# The SD conv layout reaches the layer transposed, not contiguous.
|
||||
x, cu, _ = _conv_fixture([200])
|
||||
raw = torch.full(
|
||||
(16, STATE_LEN, DIM), float("nan"), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
conv_state = raw.transpose(-1, -2)
|
||||
assert not conv_state.is_contiguous()
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([7]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(conv_state[7], x[192 - STATE_LEN : 192].transpose(0, 1))
|
||||
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
|
||||
untouched[7] = False
|
||||
assert bool(raw[untouched].isnan().all())
|
||||
|
||||
|
||||
def test_checkpoint_metadata_indexes_prefill_rows_after_decodes() -> None:
|
||||
# The chunk kernel receives the prefill tail alone, so the metadata has to
|
||||
# be indexed the same way or it snapshots the wrong request's boundary.
|
||||
md = _build(BatchSpec(seq_lens=[64, 200], query_lens=[1, 200]))
|
||||
|
||||
assert md.num_decodes == 1 and md.num_prefills == 1
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192]))
|
||||
# Request 1 owns blocks 4..7, so its slot-2 state block is 6.
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([6]))
|
||||
|
||||
|
||||
def test_conv_checkpoint_addresses_row_past_int32_limit() -> None:
|
||||
# state_stride_0 is the whole mamba page, padded up to the attention page,
|
||||
# so row * stride passes 2**31 a few thousand blocks into a large cache and
|
||||
# an int32 wrap stores outside the tensor. The margin on the plain prefill
|
||||
# path is 1.6x, which is luck: a bigger cache, a higher utilization or the
|
||||
# wider row speculative decoding allocates all close it.
|
||||
stride_row = DIM * (STATE_LEN + 8)
|
||||
row = 2**31 // stride_row + 1
|
||||
need = (row + 1) * stride_row * 2
|
||||
free, _ = torch.accelerator.get_memory_info()
|
||||
if free < need + (4 << 30):
|
||||
pytest.skip(f"needs {need >> 30} GiB of free VRAM to reach the wrap")
|
||||
|
||||
x, cu, _ = _conv_fixture([200])
|
||||
conv_state = torch.zeros(
|
||||
row + 1, DIM, STATE_LEN + 8, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
assert conv_state.stride(0) * row > 2**31, "fixture does not reach the wrap"
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([row]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(
|
||||
conv_state[row, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
|
||||
)
|
||||
assert not bool(conv_state[:row].any()), "the store wrote outside its row"
|
||||
|
||||
|
||||
def test_conv_checkpoint_reads_row_strided_projection() -> None:
|
||||
# mixed_qkv is a band of the fused QKVGFAB projection, so its row stride is
|
||||
# the whole projection width, not DIM.
|
||||
torch.manual_seed(5)
|
||||
total, pad = 200, 177
|
||||
projection = torch.randn(total, DIM + pad, device="cuda", dtype=torch.bfloat16)
|
||||
x = projection[:, :DIM]
|
||||
assert x.stride(0) == DIM + pad
|
||||
|
||||
cu = _i32([0, total])
|
||||
conv_state = torch.full(
|
||||
(16, DIM, STATE_LEN), float("nan"), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([4]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(conv_state[4], x[192 - STATE_LEN : 192].transpose(0, 1))
|
||||
|
||||
|
||||
def test_finer_prefix_match_unit_moves_checkpoint_to_hash_grid() -> None:
|
||||
# A partial prefix hit matches at prefix_match_unit, so a state captured
|
||||
# only at block boundaries is too coarse to resume from. Block 256 with
|
||||
# unit 128 moves a 700-token prompt's checkpoint from 512 to 640.
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[700], query_lens=[700]),
|
||||
block_size=256,
|
||||
prefix_match_unit=128,
|
||||
)
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([640]))
|
||||
# Slot is still block-indexed: cdiv(700, 256) - 2 = 1, request 0 owns 0..2.
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([1]))
|
||||
|
||||
|
||||
def test_checkpoint_metadata_skips_unaligned_offset() -> None:
|
||||
# The scheduler declines a boundary off the published alignment and
|
||||
# reserves nothing, so the builder has to reach the same verdict or it
|
||||
# writes a block nobody hashed.
|
||||
# Block 192, unit 96: a 700-token prompt's last hash boundary is 672, which
|
||||
# is 32 tokens off the chunk grid.
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[700], query_lens=[700]),
|
||||
block_size=192,
|
||||
prefix_match_unit=96,
|
||||
)
|
||||
|
||||
assert md.checkpoint is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Opting in
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _spec_for(monkeypatch, **attrs) -> MambaSpec:
|
||||
"""Run the layer's `get_kv_cache_spec` over a stubbed base spec."""
|
||||
base = MambaSpec(block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,))
|
||||
monkeypatch.setattr(
|
||||
GatedDeltaNetAttention, "get_kv_cache_spec", lambda self, cfg: base
|
||||
)
|
||||
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
|
||||
for name, value in dict(
|
||||
use_prefill_checkpoint=True,
|
||||
use_fused_chunk=True,
|
||||
num_spec=0,
|
||||
use_safe_gate=True,
|
||||
).items():
|
||||
setattr(layer, name, attrs.get(name, value))
|
||||
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
|
||||
vllm_config.cache_config.mamba_cache_mode = "align"
|
||||
return layer.get_kv_cache_spec(vllm_config)
|
||||
|
||||
|
||||
def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
|
||||
# The scheduler learns the kernel's granularity only from the spec, so
|
||||
# reserving a block without publishing the alignment would hash a state
|
||||
# that is never written.
|
||||
spec = _spec_for(monkeypatch)
|
||||
|
||||
assert spec.num_prefill_checkpoint_blocks == 1
|
||||
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attr,value",
|
||||
[
|
||||
("use_prefill_checkpoint", False),
|
||||
("use_fused_chunk", False),
|
||||
("use_safe_gate", False),
|
||||
],
|
||||
)
|
||||
def test_paths_that_cannot_export_stay_opted_out(monkeypatch, attr, value) -> None:
|
||||
spec = _spec_for(monkeypatch, **{attr: value})
|
||||
|
||||
assert spec.num_prefill_checkpoint_blocks == 0
|
||||
assert spec.prefill_checkpoint_alignment is None
|
||||
|
||||
|
||||
def test_speculative_decoding_does_not_opt_out(monkeypatch) -> None:
|
||||
# num_spec reaches the kernels as a wider conv row and an interleaved
|
||||
# batch, both of which the layer handles.
|
||||
spec = _spec_for(monkeypatch, num_spec=8)
|
||||
|
||||
assert spec.num_prefill_checkpoint_blocks == 1
|
||||
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
|
||||
@@ -13,17 +13,17 @@ import torch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
def _fused_chunk_available() -> bool:
|
||||
if not (current_platform.is_rocm() and torch.cuda.is_available()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import is_fused_kda_chunk_supported
|
||||
|
||||
return on_gfx950()
|
||||
return is_fused_kda_chunk_supported()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_gfx950(),
|
||||
reason="The fused KDA chunk kernel is only built for gfx950",
|
||||
not _fused_chunk_available(),
|
||||
reason="The fused KDA chunk kernel needs a gfx950 build that includes it",
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
@@ -701,6 +701,56 @@ def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None:
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_state_indices_accept_a_block_table_column(use_fused: bool) -> None:
|
||||
# The layer passes block_table[:, 0], which is contiguous only while the
|
||||
# table has one column. Speculative decoding reserves a state page per
|
||||
# draft token, so the table widens and that column becomes strided.
|
||||
_requires_kernel()
|
||||
seqlens = [512, 320, 64]
|
||||
inp = _inputs(seqlens, seed=43)
|
||||
n = len(seqlens)
|
||||
rows = [3 * i + 5 for i in range(n)]
|
||||
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
|
||||
ref_cache = cache.clone()
|
||||
|
||||
table = torch.zeros(n, 9, dtype=torch.int32, device="cuda")
|
||||
table[:, 0] = idx
|
||||
column = table[:, 0]
|
||||
assert not column.is_contiguous()
|
||||
|
||||
o_ref, _ = _run_paged(inp, ref_cache, idx, warm, use_fused=use_fused)
|
||||
o_got, _ = _run_paged(inp, cache, column, warm, use_fused=use_fused)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref)
|
||||
torch.testing.assert_close(cache, ref_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_padded_zero_length_sequence_is_inert(use_fused: bool) -> None:
|
||||
# A batch carrying speculative decodes pads its non-spec group with
|
||||
# zero-length sequences whose cache row still belongs to a request, so
|
||||
# consuming no tokens has to mean writing nothing and perturbing nobody.
|
||||
_requires_kernel()
|
||||
live = [200, 1, 128]
|
||||
# _inputs draws h0 last, so the padded batch's live rows are the same draw
|
||||
# as the unpadded one's and the two runs compare elementwise.
|
||||
padded, unpadded = _inputs([*live, 0], seed=41), _inputs(live, seed=41)
|
||||
rows = [3 * i + 5 for i in range(4)]
|
||||
|
||||
cache, idx, warm = _paged_cache(padded, [True] * 4, rows, slots=24)
|
||||
before = cache.clone()
|
||||
_run_paged(padded, cache, idx, warm, use_fused=use_fused)
|
||||
|
||||
ref_cache, ref_idx, ref_warm = _paged_cache(unpadded, [True] * 3, rows[:3], 24)
|
||||
_run_paged(unpadded, ref_cache, ref_idx, ref_warm, use_fused=use_fused)
|
||||
|
||||
assert torch.equal(cache[rows[3]], before[rows[3]]), (
|
||||
"the padded sequence's cache row was written"
|
||||
)
|
||||
torch.testing.assert_close(cache[ref_idx.long()], ref_cache[ref_idx.long()])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None:
|
||||
"""At G>1 the read and the write are different blocks of one launch."""
|
||||
|
||||
@@ -30,13 +30,17 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia import kda as nvidia_kda
|
||||
from vllm.models.kimi_k3.nvidia.kda import (
|
||||
KimiK3DeltaAttention,
|
||||
_flashinfer_kda_prefill,
|
||||
_flashkda_prefill,
|
||||
is_flashinfer_fused_kda_decode_supported,
|
||||
is_flashinfer_fused_kda_spec_decode_supported,
|
||||
is_flashinfer_recurrent_kda_prefill_supported,
|
||||
is_flashkda_supported,
|
||||
is_fused_kda_decode_supported,
|
||||
resolve_kda_spec_decode_backend,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM
|
||||
from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
|
||||
@@ -90,6 +94,25 @@ def test_kda_warmup_skips_missing_metadata(monkeypatch):
|
||||
assert layer._forward(empty, empty, empty, empty, empty) is None
|
||||
|
||||
|
||||
def test_resolve_kda_spec_decode_backend(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
|
||||
lambda *args: True,
|
||||
)
|
||||
args = (12, 128, 4, 6, torch.bfloat16, torch.bfloat16, torch.float32, -5.0)
|
||||
assert resolve_kda_spec_decode_backend("auto", *args, False) == "flashinfer"
|
||||
assert resolve_kda_spec_decode_backend("native", *args, False) == "native"
|
||||
assert resolve_kda_spec_decode_backend("flashinfer", *args, False) == "flashinfer"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
|
||||
lambda *args: False,
|
||||
)
|
||||
assert resolve_kda_spec_decode_backend("auto", *args, False) == "native"
|
||||
with pytest.raises(RuntimeError, match="packed_fused_kda_decode"):
|
||||
resolve_kda_spec_decode_backend("flashinfer", *args, False)
|
||||
|
||||
|
||||
def test_kda_recoverssm_config_state_layout():
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
@@ -1182,6 +1205,224 @@ def test_fused_kda_decode_correctness(
|
||||
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_flashinfer_fused_kda_spec_decode_integration(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
state_dtype: torch.dtype,
|
||||
):
|
||||
if not is_flashinfer_fused_kda_spec_decode_supported(
|
||||
num_heads=12,
|
||||
head_dim=128,
|
||||
conv_width=4,
|
||||
num_spec=6,
|
||||
input_dtype=torch.bfloat16,
|
||||
conv_state_dtype=torch.bfloat16,
|
||||
recurrent_state_dtype=state_dtype,
|
||||
lower_bound=-5.0,
|
||||
use_recoverssm=False,
|
||||
):
|
||||
pytest.skip("FlashInfer packed fused KDA decode is not supported")
|
||||
|
||||
generator = torch.Generator(device=DEVICE).manual_seed(20260820)
|
||||
H, D, W, T = 12, 128, 4, 7
|
||||
hidden_size = H * D
|
||||
num_rows = 10
|
||||
num_slots = 16
|
||||
|
||||
x = torch.randn(
|
||||
num_rows,
|
||||
3 * hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
conv_weight = 0.1 * torch.randn(
|
||||
3 * hidden_size,
|
||||
W,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
fused_weight = conv_weight.view(3, hidden_size, W).transpose(1, 2).contiguous()
|
||||
raw_gate = torch.randn(
|
||||
1, num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
|
||||
)
|
||||
raw_beta = torch.randn(
|
||||
1, num_rows, H, dtype=torch.bfloat16, device=DEVICE, generator=generator
|
||||
)
|
||||
output_gate = torch.randn(
|
||||
num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
|
||||
)
|
||||
A_log = torch.log(torch.arange(1, H + 1, dtype=torch.float32, device=DEVICE))
|
||||
dt_bias = torch.empty(hidden_size, dtype=torch.float32, device=DEVICE).uniform_(
|
||||
-7.0, -4.0, generator=generator
|
||||
)
|
||||
norm_weight = torch.randn(
|
||||
D, dtype=torch.float32, device=DEVICE, generator=generator
|
||||
)
|
||||
norm_eps = 1e-5
|
||||
state_indices = torch.tensor(
|
||||
[
|
||||
list(range(1, T + 1)),
|
||||
list(range(T + 1, 2 * T + 1)),
|
||||
[0] * T,
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
query_start_loc = torch.tensor([0, 7, 10, 10], dtype=torch.int32, device=DEVICE)
|
||||
num_accepted_tokens = torch.tensor([1, 4, 1], dtype=torch.int32, device=DEVICE)
|
||||
conv_seed = (
|
||||
0.5
|
||||
* torch.randn(
|
||||
num_slots,
|
||||
T + W - 2,
|
||||
3 * hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
).transpose(1, 2)
|
||||
state_seed = (
|
||||
0.5
|
||||
* torch.randn(
|
||||
num_slots,
|
||||
H,
|
||||
D,
|
||||
D,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
).to(state_dtype)
|
||||
metadata = KimiK3KDAMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_spec_decodes=2,
|
||||
num_spec_decode_tokens=num_rows,
|
||||
num_actual_tokens=num_rows,
|
||||
spec_query_start_loc=query_start_loc,
|
||||
spec_state_indices_tensor=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.nvidia.kda.get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={"test.layer": metadata}),
|
||||
)
|
||||
|
||||
def clone_strided(tensor: torch.Tensor) -> torch.Tensor:
|
||||
clone = torch.empty_strided(
|
||||
tensor.shape, tensor.stride(), dtype=tensor.dtype, device=tensor.device
|
||||
)
|
||||
clone.copy_(tensor)
|
||||
return clone
|
||||
|
||||
def composed_reference(
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
mixed_qkv = causal_conv1d_update(
|
||||
x,
|
||||
conv_state,
|
||||
conv_weight,
|
||||
bias=None,
|
||||
activation="silu",
|
||||
conv_state_indices=state_indices[:, 0],
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
max_query_len=T,
|
||||
validate_data=False,
|
||||
out=torch.empty_like(x),
|
||||
)
|
||||
q, k, v = mixed_qkv.view(num_rows, 3, H, D).unbind(1)
|
||||
recurrent_out, _ = fused_recurrent_kda_nvidia(
|
||||
q=q.unsqueeze(0),
|
||||
k=k.unsqueeze(0),
|
||||
v=v.unsqueeze(0),
|
||||
raw_g=raw_gate,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=-5.0,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=query_start_loc,
|
||||
ssm_state_indices=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
recurrent_float = recurrent_out.float()
|
||||
return (
|
||||
recurrent_float
|
||||
* torch.rsqrt(
|
||||
recurrent_float.square().mean(dim=-1, keepdim=True) + norm_eps
|
||||
)
|
||||
* norm_weight
|
||||
* output_gate.float().sigmoid().unsqueeze(0)
|
||||
).to(torch.bfloat16)
|
||||
|
||||
def run_flashinfer(
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
layer = SimpleNamespace(
|
||||
prefix="test.layer",
|
||||
kv_cache=(conv_state.transpose(-1, -2), recurrent_state),
|
||||
kda_spec_decode_backend="flashinfer",
|
||||
decode_conv1d_weight=fused_weight,
|
||||
decode_norm_weight=norm_weight,
|
||||
gate_lower_bound=-5.0,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
o_norm=SimpleNamespace(eps=norm_eps),
|
||||
)
|
||||
KimiK3DeltaAttention._forward(
|
||||
layer,
|
||||
mixed_qkv=x,
|
||||
g1=raw_gate,
|
||||
g2=output_gate,
|
||||
beta=raw_beta,
|
||||
core_attn_out=output,
|
||||
)
|
||||
return output
|
||||
|
||||
expected_conv = clone_strided(conv_seed)
|
||||
expected_state = state_seed.clone()
|
||||
expected = composed_reference(expected_conv, expected_state)
|
||||
actual_conv = clone_strided(conv_seed)
|
||||
actual_state = state_seed.clone()
|
||||
output = torch.empty_like(expected)
|
||||
actual = run_flashinfer(actual_conv, actual_state, output)
|
||||
|
||||
assert actual.data_ptr() == output.data_ptr()
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=3e-2)
|
||||
assert torch.count_nonzero(actual[:, :7]) > 0
|
||||
torch.testing.assert_close(actual_state[0], state_seed[0], atol=0, rtol=0)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = run_flashinfer(actual_conv, actual_state, output)
|
||||
|
||||
actual_conv.copy_(conv_seed)
|
||||
actual_state.copy_(state_seed)
|
||||
query_start_loc.copy_(torch.tensor([0, 6, 10, 10], device=DEVICE))
|
||||
num_accepted_tokens.copy_(torch.tensor([3, 2, 1], device=DEVICE))
|
||||
expected_conv.copy_(conv_seed)
|
||||
expected_state.copy_(state_seed)
|
||||
expected = composed_reference(expected_conv, expected_state)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert captured.data_ptr() == output.data_ptr()
|
||||
torch.testing.assert_close(output, expected, atol=2e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
|
||||
|
||||
def test_fused_kda_decode_rejects_speculative_conv_state():
|
||||
assert not is_fused_kda_decode_supported(
|
||||
num_heads=12,
|
||||
|
||||
@@ -259,7 +259,7 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
|
||||
|
||||
model_output = outputs[0].outputs[0].text.strip()
|
||||
assert model_output.startswith(tool_parser.bot_token), model_output
|
||||
parsed_message = tool_parser.extract_tool_calls(model_output, None) # type: ignore[arg-type]
|
||||
parsed_message = tool_parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert parsed_message.tools_called
|
||||
|
||||
@@ -302,7 +302,7 @@ def test_mistral_function_call_nested_json():
|
||||
|
||||
model_output = f"{parser.bot_token}get_current_weather{json.dumps(args_dict)}"
|
||||
|
||||
parsed = parser.extract_tool_calls(model_output, None) # type: ignore[arg-type]
|
||||
parsed = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
# Assertions: the tool call is detected and the full nested JSON is parsed
|
||||
# without truncation.
|
||||
@@ -335,7 +335,7 @@ def test_mistral_function_call_nested_json():
|
||||
]
|
||||
)
|
||||
|
||||
parsed = parser.extract_tool_calls(model_output, None) # type: ignore[arg-type]
|
||||
parsed = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
# Assertions: the tool call is detected and the full nested JSON is parsed
|
||||
# without truncation.
|
||||
|
||||
@@ -7,6 +7,15 @@ import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
|
||||
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
|
||||
embeddings = []
|
||||
for d in response.data:
|
||||
assert isinstance(d.embedding, list)
|
||||
embeddings.append(d.embedding)
|
||||
return embeddings
|
||||
|
||||
|
||||
def run_embedding_correctness_test(
|
||||
|
||||
@@ -104,7 +104,16 @@ def run_test(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.2"])
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
|
||||
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
|
||||
# instead of shipping a separate lm_head.weight tensor; this
|
||||
# exercises that path (see tie_weights() in nemotron_parse.py).
|
||||
"nvidia/NVIDIA-Nemotron-Parse-2.0",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from importlib.metadata import version
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
from packaging.version import Version
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
|
||||
from ....conftest import (
|
||||
IMAGE_ASSETS,
|
||||
HfRunner,
|
||||
PromptImageInput,
|
||||
VllmRunner,
|
||||
)
|
||||
from ....utils import multi_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
Version("5.0") <= Version(version("transformers")),
|
||||
reason=(
|
||||
"vllm upgraded transformers above v5.4 where HF model custom code uses siglip2 "
|
||||
"internals (filter_out_non_signature_kwargs) removed by "
|
||||
"huggingface/transformers#43514"
|
||||
),
|
||||
)
|
||||
|
||||
MODEL_ID = "microsoft/Phi-4-reasoning-vision-15B"
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|user|>\n<image>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
"cherry_blossom": "<|user|>\n<image>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
}
|
||||
)
|
||||
HF_MULTIIMAGE_IMAGE_PROMPT = (
|
||||
"<|user|>\n<image>\n<image>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
|
||||
)
|
||||
|
||||
DTYPE = "half"
|
||||
MAX_TOKENS = 128
|
||||
NUM_LOGPROBS = 10
|
||||
|
||||
|
||||
def vllm_to_hf_output(
|
||||
vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str
|
||||
):
|
||||
"""Sanitize vllm output to be comparable with hf output."""
|
||||
_, output_str, out_logprobs = vllm_output
|
||||
|
||||
output_str_without_image = re.sub(r"(<image>)+", "", output_str)
|
||||
if output_str_without_image and output_str_without_image[0] == " ":
|
||||
output_str_without_image = output_str_without_image[1:]
|
||||
|
||||
hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
|
||||
hf_output_ids = tokenizer.encode(output_str_without_image)
|
||||
if hf_output_ids and hf_output_ids[0] == tokenizer.bos_token_id:
|
||||
hf_output_ids = hf_output_ids[1:]
|
||||
|
||||
return hf_output_ids, hf_output_str, out_logprobs
|
||||
|
||||
|
||||
def _build_single_image_inputs(
|
||||
image_assets,
|
||||
) -> list[tuple[list[str], PromptImageInput]]:
|
||||
"""Build single-image inputs for all size_factors at once."""
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
all_inputs: list[tuple[list[str], PromptImageInput]] = []
|
||||
for size_factors in [[1.0], [0.25, 0.5, 1.0]]:
|
||||
for image, prompt in zip(images, HF_IMAGE_PROMPTS):
|
||||
all_inputs.append(
|
||||
(
|
||||
[prompt for _ in size_factors],
|
||||
[rescale_image_size(image, f) for f in size_factors],
|
||||
)
|
||||
)
|
||||
return all_inputs
|
||||
|
||||
|
||||
def _build_multi_image_inputs(
|
||||
image_assets,
|
||||
) -> list[tuple[list[str], PromptImageInput]]:
|
||||
"""Build multi-image inputs for all size_factors at once."""
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
all_inputs: list[tuple[list[str], PromptImageInput]] = []
|
||||
for size_factors in [[0.5], [0.15, 0.30]]:
|
||||
all_inputs.append(
|
||||
(
|
||||
[HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
|
||||
[
|
||||
[rescale_image_size(image, factor) for image in images]
|
||||
for factor in size_factors
|
||||
],
|
||||
)
|
||||
)
|
||||
return all_inputs
|
||||
|
||||
|
||||
def _run_and_compare(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
all_inputs: Sequence[tuple[list[str], PromptImageInput]],
|
||||
model: str,
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
mm_limit: int,
|
||||
gpu_memory_utilization: float,
|
||||
):
|
||||
"""Load each runner once, run all inputs, then compare."""
|
||||
# NOTE: run vLLM first, then HF. vLLM needs a fresh process without
|
||||
# cuda initialization; running HF first would break the multiprocessing
|
||||
# backend with fork method.
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=max_model_len,
|
||||
max_num_seqs=max_num_seqs,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
dtype=DTYPE,
|
||||
limit_mm_per_prompt={"image": mm_limit},
|
||||
tensor_parallel_size=2,
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
MAX_TOKENS,
|
||||
num_logprobs=NUM_LOGPROBS,
|
||||
images=images,
|
||||
)
|
||||
for prompts, images in all_inputs
|
||||
]
|
||||
|
||||
hf_model_kwargs = {"_attn_implementation": "sdpa", "device_map": "auto"}
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=DTYPE,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
trust_remote_code=True,
|
||||
) as hf_model:
|
||||
hf_outputs_per_case = [
|
||||
hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
MAX_TOKENS,
|
||||
num_logprobs=NUM_LOGPROBS,
|
||||
images=images,
|
||||
)
|
||||
for prompts, images in all_inputs
|
||||
]
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("model", [MODEL_ID])
|
||||
def test_models(hf_runner, vllm_runner, image_assets, model) -> None:
|
||||
all_inputs = _build_single_image_inputs(image_assets)
|
||||
_run_and_compare(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
all_inputs,
|
||||
model,
|
||||
max_model_len=8192,
|
||||
max_num_seqs=2,
|
||||
mm_limit=1,
|
||||
gpu_memory_utilization=0.80,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("model", [MODEL_ID])
|
||||
def test_multi_images_models(hf_runner, vllm_runner, image_assets, model) -> None:
|
||||
all_inputs = _build_multi_image_inputs(image_assets)
|
||||
_run_and_compare(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
all_inputs,
|
||||
model,
|
||||
max_model_len=8192,
|
||||
max_num_seqs=2,
|
||||
mm_limit=2,
|
||||
gpu_memory_utilization=0.80,
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSpeechSeq2Seq
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
@@ -264,6 +265,49 @@ def test_parse_language_detection_output():
|
||||
cls.parse_language_detection_output([], make_tokenizer("anything"))
|
||||
|
||||
|
||||
def test_create_fake_bias_for_k_proj_injects_out_features():
|
||||
"""Packed k_proj weights get a zeros bias of length ``out_features``."""
|
||||
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
|
||||
|
||||
packed = torch.ones(2, 16)
|
||||
result = dict(
|
||||
_create_fake_bias_for_k_proj(
|
||||
[("model.encoder.layers.0.self_attn.k_proj.weight_packed", packed)],
|
||||
".k_proj.weight_packed",
|
||||
out_features=8,
|
||||
)
|
||||
)
|
||||
bias_name = "model.encoder.layers.0.self_attn.k_proj.bias"
|
||||
assert bias_name in result
|
||||
assert "model.encoder.layers.0.self_attn.k_proj.bias_packed" not in result
|
||||
assert tuple(result[bias_name].shape) == (8,)
|
||||
assert torch.equal(result[bias_name], torch.zeros(8))
|
||||
|
||||
|
||||
def test_create_fake_bias_for_k_proj_forwards_real_bias():
|
||||
"""A checkpoint bias is forwarded and no extra fake bias is emitted."""
|
||||
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
|
||||
|
||||
weight = torch.ones(8, 16)
|
||||
real_bias = torch.ones(8)
|
||||
items = list(
|
||||
_create_fake_bias_for_k_proj(
|
||||
[
|
||||
("layers.0.self_attn.k_proj.weight", weight),
|
||||
("layers.0.self_attn.k_proj.bias", real_bias),
|
||||
],
|
||||
".k_proj.weight",
|
||||
out_features=8,
|
||||
)
|
||||
)
|
||||
names = [n for n, _ in items]
|
||||
assert names == [
|
||||
"layers.0.self_attn.k_proj.weight",
|
||||
"layers.0.self_attn.k_proj.bias",
|
||||
]
|
||||
assert items[1][1] is real_bias
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
|
||||
|
||||
@@ -1193,7 +1193,7 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
|
||||
output.sequences = output.sequences[:, prompt_len:]
|
||||
return output
|
||||
|
||||
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign, assignment]
|
||||
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign]
|
||||
hf_model.model.generate = patched_generate # type: ignore[method-assign]
|
||||
return hf_model
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import GenerateModelInfo
|
||||
|
||||
from .ppl_utils import vqa_ppl_test
|
||||
|
||||
MODELS = [
|
||||
(GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28),
|
||||
]
|
||||
|
||||
|
||||
def _pixel_size(factor: int):
|
||||
# Match max image token size with qwen's PPL test
|
||||
return {
|
||||
"size": {
|
||||
"shortest_edge": 2 * factor**2,
|
||||
"longest_edge": 2 * 1280 * factor**2,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info,factor", MODELS)
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
|
||||
def test_ppl(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: GenerateModelInfo,
|
||||
factor: int,
|
||||
mm_device_do_normalize: bool,
|
||||
):
|
||||
vqa_ppl_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
|
||||
mm_processor_kwargs=_pixel_size(factor),
|
||||
)
|
||||
@@ -132,7 +132,7 @@ def test_models(
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_cases, # type: ignore[arg-type]
|
||||
input_cases,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
@@ -23,7 +23,13 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import FusedMMInputNorm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processors.nemotron_vl import (
|
||||
SIGLIP_MEAN,
|
||||
SIGLIP_STD,
|
||||
LlamaNemotronVLEmbedImageProcessor,
|
||||
)
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
|
||||
from ....utils import ROCM_ENGINE_KWARGS
|
||||
@@ -52,6 +58,54 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
MODELS = ["nvidia/llama-nemotron-embed-vl-1b-v2"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_device_normalization_matches_cpu_transform() -> None:
|
||||
device = current_platform.device_type
|
||||
image = Image.new("RGB", (48, 32), color=(12, 128, 240))
|
||||
processor_kwargs = dict(
|
||||
image_size=32,
|
||||
min_dynamic_patch=1,
|
||||
max_dynamic_patch=1,
|
||||
dynamic_image_size=True,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
|
||||
cpu_processor = LlamaNemotronVLEmbedImageProcessor(**processor_kwargs)
|
||||
device_processor = LlamaNemotronVLEmbedImageProcessor(
|
||||
**processor_kwargs, do_rescale=False, do_normalize=False
|
||||
)
|
||||
|
||||
expected = cpu_processor._images_to_pixel_values_lst([image])[0]
|
||||
raw_pixels = device_processor._images_to_pixel_values_lst([image])[0]
|
||||
input_norm = FusedMMInputNorm(
|
||||
image_mean=list(SIGLIP_MEAN),
|
||||
image_std=list(SIGLIP_STD),
|
||||
rescale_factor=1 / 255,
|
||||
).to(device)
|
||||
actual = input_norm(
|
||||
raw_pixels.flatten(start_dim=1).to(device), expected.dtype
|
||||
).view_as(expected)
|
||||
|
||||
assert raw_pixels.dtype == torch.uint8
|
||||
torch.testing.assert_close(actual, expected.to(device))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("do_rescale", "do_normalize"), [(True, False), (False, True)])
|
||||
def test_device_normalization_rejects_partial_cpu_processing(
|
||||
do_rescale: bool, do_normalize: bool
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="both enabled or both disabled"):
|
||||
LlamaNemotronVLEmbedImageProcessor(
|
||||
image_size=32,
|
||||
min_dynamic_patch=1,
|
||||
max_dynamic_patch=1,
|
||||
dynamic_image_size=True,
|
||||
use_thumbnail=False,
|
||||
do_rescale=do_rescale,
|
||||
do_normalize=do_normalize,
|
||||
)
|
||||
|
||||
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
@@ -136,7 +190,7 @@ def test_models(
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_cases, # type: ignore[arg-type]
|
||||
input_cases,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
@@ -149,7 +149,7 @@ def test_models(
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_cases, # type: ignore[arg-type]
|
||||
input_cases,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 VL processing helpers."""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import (
|
||||
DeepseekV4VLProcessingInfo,
|
||||
DeepseekV4VLProcessor,
|
||||
build_image_block_pad_free,
|
||||
)
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
|
||||
|
||||
class _ConfigCtx:
|
||||
def __init__(self, config: DeepseekV4Config) -> None:
|
||||
self._config = config
|
||||
|
||||
def get_hf_config(self, *args, **kwargs):
|
||||
return self._config
|
||||
|
||||
|
||||
def _feature_counts(
|
||||
processor: DeepseekV4VLProcessor, width: int, height: int
|
||||
) -> tuple[int, int]:
|
||||
out = processor(images=[Image.new("RGB", (width, height))])
|
||||
n_llm_h, n_llm_w = out["llm_grid"][0].tolist()
|
||||
types, _ = build_image_block_pad_free(n_llm_h, n_llm_w)
|
||||
return len(types), out["patches"].shape[0]
|
||||
|
||||
|
||||
def test_get_image_size_with_most_features_beats_wide_real_image():
|
||||
"""Dummy size must maximize tokens/patches within the budget (#59271).
|
||||
|
||||
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
|
||||
"""
|
||||
config = DeepseekV4Config(vision_n_layers=24)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
processor = DeepseekV4VLProcessor(config)
|
||||
|
||||
dummy = info.get_image_size_with_most_features()
|
||||
dummy_features = _feature_counts(processor, dummy.width, dummy.height)
|
||||
wide_features = _feature_counts(processor, 1932, 336)
|
||||
|
||||
assert dummy == (1932, 336)
|
||||
assert wide_features <= dummy_features
|
||||
|
||||
|
||||
def test_get_image_size_with_most_features_respects_max_wh_ratio():
|
||||
config = DeepseekV4Config(vision_n_layers=24)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
dummy = info.get_image_size_with_most_features()
|
||||
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
|
||||
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Video placeholder accounting for GLM-5.3-Flash.
|
||||
|
||||
``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
|
||||
placeholders per timestamp, so the timestamps returned by
|
||||
``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
|
||||
while ``video_grid_thw`` decides how many rows the vision tower produces. If
|
||||
the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
|
||||
inside a worker and takes the engine down with it.
|
||||
|
||||
The checks below are arithmetic: no weights, no GPU.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.models.glm5_next.video_processing_glm5_next import (
|
||||
Glm5NextVideoProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
|
||||
def _pixel_path_grid(
|
||||
video_processor: Glm5NextVideoProcessor,
|
||||
num_frames: int,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> tuple[int, int, int]:
|
||||
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
|
||||
resized_height, resized_width = smart_resize(
|
||||
num_frames=num_frames,
|
||||
height=height,
|
||||
width=width,
|
||||
temporal_factor=video_processor.temporal_patch_size,
|
||||
factor=video_processor.patch_size
|
||||
* video_processor.merge_size
|
||||
* video_processor.patch_expand_factor,
|
||||
min_pixels=video_processor.min_image_tokens,
|
||||
max_pixels=video_processor.max_image_tokens,
|
||||
)
|
||||
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
|
||||
return (
|
||||
padded_frames // video_processor.temporal_patch_size,
|
||||
resized_height // video_processor.patch_size,
|
||||
resized_width // video_processor.patch_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
|
||||
[
|
||||
# 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
|
||||
(32, 8.0, 4.0, 480, 640, (4, 36, 46)),
|
||||
# 1080p, 20 s: same factor of 3 at a full-size canvas.
|
||||
(600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
|
||||
# 1080p, 60 s: the one duration window where the two samplers agree
|
||||
# anyway -- a regression guard, the count must not move.
|
||||
(1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
|
||||
# Past 300 s the GLM-4.6V sampler asks for half as many instead.
|
||||
(9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
|
||||
],
|
||||
)
|
||||
def test_video_placeholders_match_encoder_rows(
|
||||
processor,
|
||||
total_num_frames: int,
|
||||
fps: float,
|
||||
duration: float,
|
||||
height: int,
|
||||
width: int,
|
||||
expected_grid: tuple[int, int, int],
|
||||
):
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
frame_indices = video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
|
||||
)
|
||||
grid_t, grid_h, grid_w = _pixel_path_grid(
|
||||
video_processor, len(frame_indices), height, width
|
||||
)
|
||||
assert (grid_t, grid_h, grid_w) == expected_grid
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": total_num_frames,
|
||||
"fps": fps,
|
||||
"duration": duration,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
total_num_frames,
|
||||
)
|
||||
|
||||
merge_length = video_processor.merge_size**2
|
||||
tokens_per_frame = grid_h * grid_w // merge_length
|
||||
encoder_rows = grid_t * grid_h * grid_w // merge_length
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
assert len(timestamps) * tokens_per_frame == encoder_rows
|
||||
assert timestamps == sorted(timestamps)
|
||||
assert timestamps[0] == 0
|
||||
assert timestamps[-1] <= duration
|
||||
|
||||
|
||||
def test_video_placeholders_match_encoder_rows_when_presampled(processor):
|
||||
"""The loader may pre-sample and hand the frames over as they are."""
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
num_frames = 32
|
||||
grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 256,
|
||||
"fps": 8.0,
|
||||
"duration": 32.0,
|
||||
"do_sample_frames": False,
|
||||
"frames_indices": list(range(0, 256, 256 // num_frames)),
|
||||
},
|
||||
num_frames,
|
||||
)
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
|
||||
|
||||
def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
|
||||
"""A clip the sampler cannot pick a single frame from is a bad request."""
|
||||
with pytest.raises(ValueError, match="selected no frames"):
|
||||
processor.info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 2,
|
||||
"fps": 8.0,
|
||||
"duration": 0.25,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
)
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (310, 470), color=(17, 89, 231)),
|
||||
Image.new("RGB", (480, 320), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
@@ -4,14 +4,29 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor, BatchFeature
|
||||
from transformers.models.pixtral import PixtralProcessor
|
||||
|
||||
from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
build_mm_input_norm,
|
||||
)
|
||||
from vllm.model_executor.models.lightonocr import (
|
||||
LightOnOCRForConditionalGeneration,
|
||||
LightOnOCRProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
|
||||
from vllm.model_executor.models.pixtral import PixtralHFEncoderInfo
|
||||
from vllm.model_executor.models.pixtral import (
|
||||
PixtralHFEncoderInfo,
|
||||
pixtral_patch_embed,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItems
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
@@ -21,6 +36,65 @@ _MODEL_CONFIG_KWARGS = {"config_format": "hf"}
|
||||
_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||
|
||||
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
|
||||
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
|
||||
_RGB_RESCALE = 1.0 / 255.0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TRITON or not current_platform.is_cuda(), reason="CUDA Triton kernel"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("normalize", [False, True])
|
||||
@pytest.mark.parametrize(("channels", "patch_shape"), [(3, (4, 4)), (2, (2, 4))])
|
||||
def test_pixtral_patch_embed_preserves_image_order_and_strides(
|
||||
dtype: torch.dtype,
|
||||
normalize: bool,
|
||||
channels: int,
|
||||
patch_shape: tuple[int, int],
|
||||
):
|
||||
"""Packed projection matches CHW normalization and patch convolution."""
|
||||
device = torch.device(current_platform.device_type)
|
||||
norm = (
|
||||
FusedMMInputNorm(
|
||||
_RGB_MEAN[:channels],
|
||||
_RGB_STD[:channels],
|
||||
_RGB_RESCALE,
|
||||
channel=channels,
|
||||
).to(device)
|
||||
if normalize
|
||||
else IdentityInputNorm()
|
||||
)
|
||||
weight = torch.randn(32, channels, *patch_shape, device=device, dtype=dtype)
|
||||
images = [
|
||||
torch.randint(0, 256, (channels, 17, 18), device=device, dtype=torch.uint8),
|
||||
torch.randint(0, 256, (channels, 25, 17), device=device, dtype=torch.uint8)[
|
||||
:, 1:17, 1:17
|
||||
],
|
||||
torch.randint(0, 256, (channels, 19, 20), device=device, dtype=torch.uint8)[
|
||||
:, 1:17, 2:18
|
||||
],
|
||||
]
|
||||
reference = []
|
||||
for image in images:
|
||||
if normalize:
|
||||
normalized = (
|
||||
image.float() * norm.weight[:, None, None] + norm.bias[:, None, None]
|
||||
)
|
||||
else:
|
||||
normalized = image
|
||||
reference.append(
|
||||
F.conv2d(normalized.to(dtype).unsqueeze(0), weight, stride=patch_shape)
|
||||
)
|
||||
packed, actual = pixtral_patch_embed(images, weight, norm)
|
||||
for expected, result in zip(reference, actual):
|
||||
torch.testing.assert_close(result, expected, rtol=0.01, atol=0.02)
|
||||
expected_packed = torch.cat(
|
||||
[result.flatten(2).transpose(1, 2) for result in reference], dim=1
|
||||
)
|
||||
torch.testing.assert_close(packed, expected_packed, rtol=0.01, atol=0.02)
|
||||
|
||||
|
||||
def _process_images_with_hf(
|
||||
hf_processor,
|
||||
@@ -128,6 +202,9 @@ def test_processor_size_override(
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
vllm_hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
assert isinstance(vllm_hf_processor, PixtralProcessor)
|
||||
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
model_id,
|
||||
fix_mistral_regex=True,
|
||||
@@ -153,6 +230,53 @@ def test_processor_size_override(
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (31, 47), color=(17, 89, 231)),
|
||||
Image.new("RGB", (48, 32), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = [processor.info.get_hf_config().image_token_index] * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert all(value.dtype == torch.uint8 for value in raw_values)
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config)
|
||||
patch_size = processor.info.get_hf_config().vision_config.patch_size
|
||||
|
||||
def pack_patches(image: torch.Tensor) -> torch.Tensor:
|
||||
channels, height, width = image.shape
|
||||
rows = height // patch_size
|
||||
cols = width // patch_size
|
||||
return (
|
||||
image[:, : rows * patch_size, : cols * patch_size]
|
||||
.reshape(channels, rows, patch_size, cols, patch_size)
|
||||
.permute(1, 3, 0, 2, 4)
|
||||
.reshape(-1, channels * patch_size**2)
|
||||
)
|
||||
|
||||
for raw, normalized in zip(raw_values, normalized_values):
|
||||
output = input_norm(pack_patches(raw), normalized.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, pack_patches(normalized), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def test_scoped_request_size_overrides_configured_flat_size():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
@@ -188,6 +312,8 @@ def test_scoped_request_size_overrides_configured_flat_size():
|
||||
|
||||
|
||||
def test_lightonocr_keeps_vision_config_image_size():
|
||||
assert not LightOnOCRForConditionalGeneration.supports_mm_device_do_normalize
|
||||
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
mm_processor_kwargs={"size": {"longest_edge": 1008}},
|
||||
|
||||
@@ -106,7 +106,7 @@ def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedenc
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None)
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
@@ -236,7 +236,7 @@ def test_get_vision_info_reads_merged_modality_scope(
|
||||
do_resize=False,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
modality=modality, # type: ignore[arg-type]
|
||||
modality=modality,
|
||||
)
|
||||
|
||||
assert seen == [merged[scope]]
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend.
|
||||
|
||||
MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias
|
||||
(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which
|
||||
``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``.
|
||||
``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not
|
||||
support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose
|
||||
device gate accepts the card, so the checkpoint could not be served natively at
|
||||
all.
|
||||
|
||||
aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus
|
||||
a correction bias), so the backend only has to ask for it. This module covers
|
||||
both halves: the gate accepts the router family, and the routing the wrapper
|
||||
requests reproduces the model's own math.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
|
||||
MIMO_EXPERTS = 256 # MiMo-V2.6-Flash
|
||||
MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro
|
||||
MIMO_TOP_K = 8
|
||||
|
||||
|
||||
def mimo_routing_method() -> RoutingMethodType:
|
||||
"""MiMo-V2.6: sigmoid + correction bias, a single expert group."""
|
||||
return get_routing_method_type(
|
||||
scoring_func="sigmoid",
|
||||
top_k=MIMO_TOP_K,
|
||||
renormalize=True,
|
||||
num_expert_group=1,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
|
||||
|
||||
def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=MIMO_TOP_K,
|
||||
hidden_dim=4096,
|
||||
intermediate_size=2048,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
routing_method=mimo_routing_method(),
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def test_ungrouped_sigmoid_router_is_deepseekv3() -> None:
|
||||
"""Guard the premise of the fix: MiMo's router is DeepSeekV3-classified."""
|
||||
assert mimo_routing_method() == RoutingMethodType.DeepSeekV3
|
||||
|
||||
|
||||
def test_aiter_w4a16_supports_mimo_routing() -> None:
|
||||
"""The gate accepts MiMo's router family — the reported construction abort.
|
||||
|
||||
Only the predicate the fix touches is exercised, so this runs on any host;
|
||||
the full config path is covered by the test below.
|
||||
"""
|
||||
assert AiterW4A16ExpertsMonolithic._supports_routing_method(
|
||||
RoutingMethodType.DeepSeekV3, kMxfp4Static, None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The same gate through the real config path, device check included."""
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
mimo_moe_config(),
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert supported, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None:
|
||||
"""The routing the wrapper requests must equal MiMo's own router math.
|
||||
|
||||
MiMo's router is ungrouped: sigmoid scores, selection on
|
||||
``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then
|
||||
renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro.
|
||||
|
||||
The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20).
|
||||
"""
|
||||
aiter_routing = pytest.importorskip(
|
||||
"aiter.ops.triton.moe.moe_routing.routing"
|
||||
).routing
|
||||
|
||||
torch.manual_seed(0)
|
||||
top_k, num_tokens = MIMO_TOP_K, 16
|
||||
logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0
|
||||
bias = torch.randn(num_experts, device="cuda") * 0.5
|
||||
|
||||
# Reference: select on the biased score, return the unbiased score, renorm.
|
||||
scores = torch.sigmoid(logits.float())
|
||||
ref_ids = (scores + bias).topk(top_k, dim=-1).indices
|
||||
ref_weights = scores.gather(1, ref_ids)
|
||||
ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
routing_data, topk_indx, _ = aiter_routing(
|
||||
logits,
|
||||
top_k,
|
||||
score_mode=score_mode,
|
||||
bias=bias,
|
||||
renorm=True,
|
||||
routed_scaling_factor=1.0,
|
||||
use_grouped_topk=False,
|
||||
)
|
||||
# `topk_indx[i]` is the token-slot index (token * top_k + slot) of the
|
||||
# i-th entry of the expert-sorted array whose weights are gate_scal.
|
||||
hist = routing_data.expt_hist.to(torch.long).cpu()
|
||||
expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist)
|
||||
slot = topk_indx.reshape(-1).to(torch.long).cpu()
|
||||
weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu()
|
||||
ids = torch.full((num_tokens, top_k), -1, dtype=torch.long)
|
||||
out = torch.zeros(num_tokens, top_k)
|
||||
for i in range(slot.numel()):
|
||||
token, position = int(slot[i]) // top_k, int(slot[i]) % top_k
|
||||
ids[token, position] = int(expert_of_slot[i])
|
||||
out[token, position] = float(weights[i])
|
||||
return ids, out
|
||||
|
||||
got_ids, got_weights = routed("sigmoid")
|
||||
|
||||
# aiter's triton top-k emits each token's selection sorted by *ascending
|
||||
# expert id*, while the reference is in descending ``sigmoid(logits)+bias``
|
||||
# rank order — align both sides on the expert id so the weight check
|
||||
# becomes elementwise and still validates the expert<->weight pairing.
|
||||
def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
order = torch.argsort(ids, dim=-1)
|
||||
return ids.gather(1, order), w.gather(1, order)
|
||||
|
||||
got_ids, got_weights = align(got_ids, got_weights)
|
||||
ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu())
|
||||
assert torch.equal(got_ids, ref_ids)
|
||||
assert torch.allclose(got_weights, ref_weights, atol=1e-3)
|
||||
|
||||
# Control: the correction bias must steer selection only. Ignoring it would
|
||||
# pick from the plain sigmoid scores instead.
|
||||
plain_ids = scores.topk(top_k, dim=-1).indices
|
||||
assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids)
|
||||
@@ -10,6 +10,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
@@ -17,10 +18,6 @@ from vllm.model_executor.models.config import (
|
||||
Qwen4ExpForConditionalGenerationConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.mtp import make_mtp_hidden_buffer
|
||||
from vllm.models.qwen4_exp.config import (
|
||||
Qwen4ExpConfig,
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
@@ -42,7 +39,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
"linear_num_value_heads": 2,
|
||||
"linear_key_head_dim": 8,
|
||||
"linear_value_head_dim": 8,
|
||||
"num_experts": 0,
|
||||
"num_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
# `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
|
||||
"eos_token_id": 1,
|
||||
"hc_count": 2,
|
||||
"hc_lowrank": 4,
|
||||
"ple_layer_ids": [1],
|
||||
@@ -287,20 +287,6 @@ def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||
|
||||
|
||||
def test_text_config_normalizes_transformers_sparse_attention_spelling():
|
||||
"""Checkpoints re-exported through transformers' Qwen4Exp config class
|
||||
serialize "qwen_sparse_attention"; vLLM keys QSA off "full_attention"."""
|
||||
from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
|
||||
|
||||
config = Qwen4ExpTextConfig(
|
||||
hc_count=4,
|
||||
layer_types=["linear_attention", "qwen_sparse_attention"],
|
||||
num_hidden_layers=2,
|
||||
)
|
||||
|
||||
assert config.layer_types == ["linear_attention", "full_attention"]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _captured_transformers_warnings() -> Iterator[list[str]]:
|
||||
"""Records what transformers logs, without relying on propagation.
|
||||
|
||||
@@ -15,6 +15,7 @@ from safetensors import safe_open
|
||||
from safetensors.torch import load, load_file, save_file
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
@@ -39,7 +40,6 @@ from vllm.models.qwen4_exp.common.ple import (
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
|
||||
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
@@ -616,6 +616,32 @@ def test_pinned_embedding_forward_finalizes_prefetched_output(
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None:
|
||||
"""Forward before any start_prefetch must fail loudly, not read garbage."""
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = None
|
||||
embedding._prefetch_stream = None
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="prior start_prefetch"):
|
||||
embedding(hidden_states)
|
||||
|
||||
|
||||
def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None:
|
||||
"""_finalize_prefetch before any start_prefetch must fail loudly."""
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = None
|
||||
embedding._prefetch_stream = None
|
||||
embedding.tp_size = 1
|
||||
output = torch.zeros(2, 6, dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="prior start_prefetch"):
|
||||
embedding._finalize_prefetch(torch.empty(4, 2, 3), output)
|
||||
|
||||
|
||||
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
@@ -730,6 +756,67 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""start_prefetch lazily allocates the stream/buffer, then forward consumes it.
|
||||
|
||||
Exercises the full pinned-prefetch contract end to end: first call
|
||||
allocates the side stream and buffer with the constructor geometry,
|
||||
and forward finalizes the lookup into the returned output.
|
||||
"""
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=Qwen4ExpPLEUnquantizedEmbeddingMethod(),
|
||||
num_ngram_heads=2,
|
||||
max_total_tokens=4,
|
||||
)
|
||||
assert embedding._prefetch_buffer is None
|
||||
assert embedding._prefetch_stream is None
|
||||
|
||||
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
|
||||
loaded_weight = loaded_weight.to(torch.bfloat16)
|
||||
copy_ple_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0")
|
||||
embedding.start_prefetch(hidden_states, ngram_ids)
|
||||
|
||||
buffer = embedding._prefetch_buffer
|
||||
assert buffer is not None
|
||||
assert buffer.shape == (4, 2, 3)
|
||||
assert buffer.dtype == torch.bfloat16
|
||||
assert buffer.device.type == "cuda"
|
||||
assert embedding._prefetch_stream is not None
|
||||
|
||||
output = embedding(hidden_states)
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptMixedPrecisionConfig.from_config(
|
||||
@@ -2355,6 +2442,25 @@ def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
def test_amd_pinned_embedding_prefetch_machinery_is_lazy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The pinned embedding must not allocate the prefetch stream/buffer in init.
|
||||
|
||||
Paths that never prefetch (AMD uses the synchronous pinned-lookup op)
|
||||
keep the stream/buffer unallocated for the process lifetime. NVIDIA's
|
||||
first ``start_prefetch`` — the eager profile run, always before cudagraph
|
||||
capture — allocates them lazily.
|
||||
"""
|
||||
module, _ = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False
|
||||
)
|
||||
embedding = module.ngram_embedding
|
||||
assert embedding._prefetch_buffer is None
|
||||
assert embedding._prefetch_stream is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
|
||||
def test_amd_pinned_embedding_output_written_under_compile(
|
||||
|
||||
+87
-6
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the fused QSA pre-indexer."""
|
||||
"""Correctness tests for the fused QSA prepare kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -15,9 +15,7 @@ from vllm.models.qwen4_exp.nvidia.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa_prepare import qsa_prepare
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
@@ -36,6 +34,7 @@ ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
@@ -61,6 +60,84 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
|
||||
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
|
||||
|
||||
|
||||
def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict:
|
||||
"""Random main-attention arguments with a paged cache in vLLM's layout."""
|
||||
num_blocks = num_tokens // MAIN_PAGE + 2
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
MAIN_HK,
|
||||
MAIN_PAGE,
|
||||
2 * MAIN_D,
|
||||
dtype=torch.uint8 if fp8_cache else torch.bfloat16,
|
||||
device="cuda",
|
||||
).transpose(1, 2)
|
||||
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
|
||||
slots[-1] = -1
|
||||
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
|
||||
return dict(
|
||||
main_qkv=torch.randn(
|
||||
num_tokens,
|
||||
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
),
|
||||
main_q_norm_weight=norm_weights[0],
|
||||
main_k_norm_weight=norm_weights[1],
|
||||
main_eps=EPS,
|
||||
main_kv_cache=cache.view(torch.float8_e4m3fn) if fp8_cache else cache,
|
||||
main_slot_mapping=slots,
|
||||
main_k_scale=0.5,
|
||||
main_v_scale=2.0,
|
||||
)
|
||||
|
||||
|
||||
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
|
||||
"""Compare with fused_qk_rmsnorm_rope_gate + reshape_and_cache_flash."""
|
||||
from vllm._custom_ops import reshape_and_cache_flash
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import (
|
||||
fused_qk_rmsnorm_rope_gate,
|
||||
)
|
||||
|
||||
q_gate, k, v = main["main_qkv"].split(
|
||||
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
|
||||
)
|
||||
q, k, gate = fused_qk_rmsnorm_rope_gate(
|
||||
q_gate,
|
||||
k,
|
||||
main["main_q_norm_weight"],
|
||||
main["main_k_norm_weight"],
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
EPS,
|
||||
MAIN_HQ,
|
||||
MAIN_HK,
|
||||
MAIN_D,
|
||||
rope.rotary_dim,
|
||||
mrope_section=MROPE_SECTION if positions.ndim == 2 else None,
|
||||
norm_beta=1.0,
|
||||
)
|
||||
kv_cache = main["main_kv_cache"]
|
||||
fp8_cache = kv_cache.dtype == torch.float8_e4m3fn
|
||||
cache = torch.zeros_like(kv_cache.view(torch.uint8) if fp8_cache else kv_cache)
|
||||
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
|
||||
reshape_and_cache_flash(
|
||||
k.view(-1, MAIN_HK, MAIN_D),
|
||||
v.view(-1, MAIN_HK, MAIN_D),
|
||||
key_cache,
|
||||
value_cache,
|
||||
main["main_slot_mapping"],
|
||||
"fp8" if fp8_cache else "auto",
|
||||
torch.tensor(main["main_k_scale"], device="cuda"),
|
||||
torch.tensor(main["main_v_scale"], device="cuda"),
|
||||
)
|
||||
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(gate_out.flatten(1), gate)
|
||||
if fp8_cache:
|
||||
assert_fp8_within_one_ulp(kv_cache, cache.view(torch.float8_e4m3fn))
|
||||
else:
|
||||
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@@ -86,7 +163,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
def test_qsa_fused_prepare_matches_unfused(
|
||||
indexer_dtype,
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
@@ -246,7 +323,9 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
|
||||
qsa_pre_indexer(
|
||||
# The main-attention cache follows the recipe's pairing with the indexer.
|
||||
main = _make_main_inputs(num_tokens, indexer_dtype == torch.float8_e4m3fn)
|
||||
main_q_out, main_gate_out = qsa_prepare(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
@@ -266,7 +345,9 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
**main,
|
||||
)
|
||||
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = gemma_rmsnorm(
|
||||
@@ -56,16 +56,24 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
rope_position_offset=0,
|
||||
),
|
||||
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
|
||||
use_fused_pre_indexer=True,
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
|
||||
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
|
||||
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
|
||||
compress_ratio=2,
|
||||
)
|
||||
attn = SimpleNamespace(
|
||||
use_fused_qsa_prepare=True,
|
||||
kv_cache=torch.empty(0, 1, 1, 2),
|
||||
kv_cache_dtype="auto",
|
||||
q_norm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
|
||||
k_norm=SimpleNamespace(weight=torch.ones(1)),
|
||||
_k_scale_float=1.0,
|
||||
_v_scale_float=1.0,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
"qsa_prepare",
|
||||
lambda *args, **kwargs: updates.append((args, kwargs)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
@@ -79,11 +87,14 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
lambda *args, **kwargs: selections.append((args, kwargs)),
|
||||
)
|
||||
|
||||
actual = indexer_qsa.QSAIndexer.forward(
|
||||
actual, _ = indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 2),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
attn=attn,
|
||||
qkv=torch.zeros(2, 4),
|
||||
slot_mapping=torch.arange(2),
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
@@ -573,7 +584,6 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
skip_topk=True,
|
||||
index_kv_heads=1,
|
||||
use_fused_pre_indexer=False,
|
||||
index_n_heads=1,
|
||||
index_head_dim=64,
|
||||
indexer_dtype=torch.bfloat16,
|
||||
@@ -596,6 +606,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
|
||||
torch.zeros(8, dtype=torch.long, device=device),
|
||||
torch.full((5, 5), -1, dtype=torch.int32, device=device),
|
||||
attn=SimpleNamespace(use_fused_qsa_prepare=False),
|
||||
)
|
||||
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
|
||||
expected_compressed = torch.zeros_like(compressed_cache)
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DecoderReplayLayers runs the replay layers on the forward's replay batch."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import (
|
||||
ForwardContext,
|
||||
get_forward_context,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
NUM_TOKENS = 401
|
||||
HC, HIDDEN = 2, 3
|
||||
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
|
||||
WINDOW = 128
|
||||
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
|
||||
|
||||
|
||||
def _context(metadata):
|
||||
return ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata=metadata,
|
||||
slot_mapping={},
|
||||
is_padding=torch.zeros(NUM_TOKENS, dtype=torch.bool, device=DEVICE),
|
||||
)
|
||||
|
||||
|
||||
def _set_replay_batch(layers, rows, metadata):
|
||||
layers.rows = torch.tensor(rows, device=DEVICE)
|
||||
layers.forward_context = ForwardContext(
|
||||
no_compile_layers={}, attn_metadata=metadata, slot_mapping={}
|
||||
)
|
||||
|
||||
|
||||
def _states(num_tokens: int):
|
||||
"""(hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual),
|
||||
shaped as the model's: hidden states [T, H], mixes [T, HC], residual [T, HC, H]."""
|
||||
hidden = torch.arange(num_tokens, device=DEVICE, dtype=torch.float32)
|
||||
hidden = hidden[:, None].expand(num_tokens, HIDDEN).contiguous()
|
||||
mix = hidden[:, :HC].contiguous()
|
||||
residual = hidden[:, None, :].expand(num_tokens, HC, HIDDEN).contiguous()
|
||||
return (
|
||||
hidden,
|
||||
torch.arange(num_tokens, device=DEVICE),
|
||||
None,
|
||||
mix + 1,
|
||||
mix + 2,
|
||||
mix + 3,
|
||||
residual,
|
||||
)
|
||||
|
||||
|
||||
def test_run_gathers_states_and_realigns_shared_indexer_buffers():
|
||||
topk = torch.arange(NUM_TOKENS * 4, device=DEVICE).view(NUM_TOKENS, 4).int()
|
||||
candidates = torch.arange(NUM_TOKENS * 3, device=DEVICE).view(NUM_TOKENS, 3).int()
|
||||
topk_before, candidates_before = topk.clone(), candidates.clone()
|
||||
seen = {}
|
||||
|
||||
def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual):
|
||||
seen["hidden_states"] = hidden_states.clone()
|
||||
replay_context = get_forward_context()
|
||||
seen["attn_metadata"] = replay_context.attn_metadata
|
||||
seen["is_padding"] = replay_context.is_padding
|
||||
return residual, pre_mix
|
||||
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates])
|
||||
states = _states(NUM_TOKENS)
|
||||
hidden, residual = states[0], states[-1]
|
||||
full, sub = object(), object()
|
||||
_set_replay_batch(layers, REPLAY_ROWS, sub)
|
||||
context = _context(full)
|
||||
with override_forward_context(context):
|
||||
outputs = layers(*states)
|
||||
assert get_forward_context() is context
|
||||
|
||||
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
|
||||
assert torch.equal(seen["hidden_states"], hidden[rows])
|
||||
assert seen["attn_metadata"] is sub and seen["is_padding"] is None
|
||||
assert torch.equal(topk[: len(REPLAY_ROWS)], topk_before[rows])
|
||||
assert torch.equal(candidates[: len(REPLAY_ROWS)], candidates_before[rows])
|
||||
assert len(outputs) == 2 and outputs[0].shape == residual.shape
|
||||
assert torch.equal(outputs[0][rows], residual[rows])
|
||||
assert outputs[0][1:173].abs().sum() == 0
|
||||
|
||||
|
||||
def test_no_replay_batch_runs_the_whole_batch():
|
||||
seen = {}
|
||||
|
||||
def run_layers(hidden_states, *rest):
|
||||
seen["attn_metadata"] = get_forward_context().attn_metadata
|
||||
return (hidden_states,)
|
||||
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [])
|
||||
states = _states(NUM_TOKENS)
|
||||
full = object()
|
||||
with override_forward_context(_context(full)):
|
||||
(output,) = layers(*states)
|
||||
assert output is states[0] and seen["attn_metadata"] is full
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The decoder-side SWA bounded replay batch DeepseekV41ModelState prepares."""
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the replay buffers live on the GPU"
|
||||
)
|
||||
|
||||
WINDOW = 128
|
||||
DEVICE = torch.device("cuda")
|
||||
GROUPS = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(
|
||||
layer_names=["swa"],
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
prefix_cacheable=False, prefix_replay_tokens=WINDOW
|
||||
),
|
||||
),
|
||||
SimpleNamespace(
|
||||
layer_names=["mla"],
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
prefix_cacheable=True, prefix_replay_tokens=0
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
BLOCK_TABLES = (torch.zeros(4, 4, device=DEVICE),) * 2
|
||||
|
||||
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
|
||||
QUERY_LENS = [1, 300, 100]
|
||||
SEQ_LENS = [500, 300, 300]
|
||||
PREFILLING = [False, True, True]
|
||||
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def state(monkeypatch):
|
||||
"""A DeepseekV41ModelState whose attention builds are recorded, not run."""
|
||||
cfg = MagicMock()
|
||||
cfg.model_config.enable_prompt_embeds = False
|
||||
cfg.model_config.uses_mrope = False
|
||||
cfg.model_config.is_multimodal_model = False
|
||||
cfg.scheduler_config.max_num_seqs = 16
|
||||
cfg.scheduler_config.max_num_batched_tokens = 1024
|
||||
cfg.parallel_config.data_parallel_size = 1
|
||||
cfg.compilation_config.fast_moe_cold_start = False
|
||||
layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None)
|
||||
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
|
||||
builds: list = []
|
||||
|
||||
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
|
||||
build = SimpleNamespace(
|
||||
batch=input_batch,
|
||||
cg_mode=cg_mode,
|
||||
slot_mappings=slot_mappings.clone(),
|
||||
replay_start=kw["model_specific_attn_metadata"].replay_start,
|
||||
attn_metadata=object(),
|
||||
)
|
||||
builds.append(build)
|
||||
return {"swa": build.attn_metadata}
|
||||
|
||||
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
|
||||
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
|
||||
state.builds = builds # type: ignore[attr-defined]
|
||||
return state
|
||||
|
||||
|
||||
def _input_batch(
|
||||
query_lens: list[int],
|
||||
seq_lens: list[int],
|
||||
is_prefilling: list[bool],
|
||||
device_query_lens: list[int] | None = None,
|
||||
num_tokens_after_padding: int | None = None,
|
||||
max_query_len: int | None = None,
|
||||
) -> InputBatch:
|
||||
num_reqs, num_tokens = len(query_lens), sum(query_lens)
|
||||
num_padded = num_tokens_after_padding or num_tokens
|
||||
batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(16, 1024, DEVICE))
|
||||
return replace(
|
||||
batch,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_padded,
|
||||
query_start_loc=torch.tensor(
|
||||
[0, *np.cumsum(device_query_lens or query_lens)],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
),
|
||||
query_start_loc_np=np.array([0, *np.cumsum(query_lens)], dtype=np.int32),
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE),
|
||||
seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32),
|
||||
is_prefilling_np=np.array(is_prefilling, dtype=np.bool_),
|
||||
positions=torch.cat(
|
||||
[torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)]
|
||||
+ [torch.zeros(num_padded - num_tokens, dtype=torch.int64)]
|
||||
).to(DEVICE),
|
||||
is_padding=torch.arange(num_padded, device=DEVICE) >= num_tokens,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
|
||||
|
||||
def _slot_mappings(num_tokens: int) -> torch.Tensor:
|
||||
return torch.stack([torch.arange(num_tokens), torch.arange(num_tokens) * 10]).to(
|
||||
DEVICE
|
||||
)
|
||||
|
||||
|
||||
def _prepare(state, batch, cg_mode):
|
||||
"""Runs prepare_attn; returns the replay batch and the replay build's inputs."""
|
||||
state.prepare_attn(
|
||||
batch,
|
||||
cg_mode,
|
||||
BLOCK_TABLES,
|
||||
_slot_mappings(batch.num_tokens_after_padding),
|
||||
[],
|
||||
GROUPS,
|
||||
)
|
||||
layers = state.decoder_replay_layers
|
||||
if layers.rows is None:
|
||||
assert layers.forward_context is None
|
||||
return None, None
|
||||
return SimpleNamespace(
|
||||
rows=layers.rows, forward_context=layers.forward_context
|
||||
), state.builds[-1]
|
||||
|
||||
|
||||
def test_replay_batch_keeps_each_request_window(state):
|
||||
state._replay_start_np[2] = 50 # the encoder-side replay start of request 2
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
|
||||
assert torch.equal(replay.rows, rows)
|
||||
sub = build.batch
|
||||
assert build.cg_mode == CUDAGraphMode.NONE
|
||||
assert sub.query_start_loc.tolist() == [0, 1, 129, 229]
|
||||
assert sub.query_start_loc_np.tolist() == [0, 1, 129, 229]
|
||||
assert sub.num_tokens == sub.num_tokens_after_padding == 229
|
||||
assert sub.max_query_len == WINDOW
|
||||
# The trimmed request's window starts at its replay window; a higher
|
||||
# encoder-side replay start stands.
|
||||
assert build.replay_start.tolist() == [0, 300 - WINDOW, 50]
|
||||
assert torch.equal(sub.positions, batch.positions[rows])
|
||||
assert torch.equal(build.slot_mappings, _slot_mappings(401)[:, rows])
|
||||
context = replay.forward_context
|
||||
assert context.attn_metadata == {"swa": build.attn_metadata}
|
||||
assert torch.equal(context.slot_mapping["mla"], _slot_mappings(401)[1, rows])
|
||||
assert context.dp_metadata is None and context.is_padding is None
|
||||
|
||||
|
||||
def test_replay_batch_keeps_device_decode_boundaries(state):
|
||||
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
|
||||
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
|
||||
batch = _input_batch(
|
||||
[2, 2, 300], [10, 10, 300], PREFILLING, device_query_lens=[1, 3, 300]
|
||||
)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == [0, 1, 2, 3, *range(304 - WINDOW, 304)]
|
||||
assert build.batch.query_start_loc.tolist() == [0, 1, 4, 4 + WINDOW]
|
||||
|
||||
|
||||
def test_prompt_logprobs_requests_keep_their_rows(state):
|
||||
"""Every prompt row of a prompt-logprobs request is read, so it never
|
||||
trims; the other prefills still do."""
|
||||
state._keeps_rows_np[1] = True
|
||||
batch = _input_batch([1, 300, 300], [500, 300, 300], PREFILLING)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == [0, *range(1, 301), *range(601 - WINDOW, 601)]
|
||||
assert build.batch.max_query_len == 300
|
||||
|
||||
|
||||
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
|
||||
"""Adaptive verification bounds the decodes' device-side query lengths
|
||||
above their CPU lengths; the replay batch keeps that bound."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, max_query_len=200)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None and build.batch.max_query_len == 200
|
||||
|
||||
|
||||
def test_only_eager_steps_trim(state):
|
||||
"""A CUDA graph keeps the layers on its whole batch; an eager step trims a
|
||||
padded batch to its real rows."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
|
||||
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
|
||||
assert _prepare(state, batch, cg_mode) == (None, None)
|
||||
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == REPLAY_ROWS
|
||||
assert build.cg_mode == CUDAGraphMode.NONE
|
||||
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
|
||||
|
||||
|
||||
def test_whole_batch_forwards_get_no_replay_batch(state):
|
||||
short = _input_batch([100, 100], [100, 100], [True, True])
|
||||
for cg_mode in (CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
|
||||
assert _prepare(state, short, cg_mode) == (None, None)
|
||||
assert len(state.builds) == 3 # the batch's own metadata only
|
||||
# Dummy batches, captures included (prepared as eager), are not prefills and
|
||||
# keep their rows.
|
||||
dummy = _input_batch(
|
||||
QUERY_LENS, SEQ_LENS, [False] * 3, num_tokens_after_padding=512
|
||||
)
|
||||
assert _prepare(state, dummy, CUDAGraphMode.NONE) == (None, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dp_state(state, monkeypatch):
|
||||
"""`state` on DP rank 0 of 2; `state.other` sets what rank 1 reports
|
||||
(trims, replay tokens)."""
|
||||
import vllm.models.deepseek_v41.nvidia.model_state as module
|
||||
|
||||
state.vllm_config.parallel_config.data_parallel_size = 2
|
||||
state.vllm_config.parallel_config.data_parallel_rank = 0
|
||||
state.vllm_config.parallel_config.is_moe_model = True
|
||||
state.other = (False, 0) # type: ignore[attr-defined]
|
||||
|
||||
def all_reduce(tensor, group):
|
||||
tensor[:, 1] = torch.tensor(state.other, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(module.dist, "all_reduce", all_reduce)
|
||||
monkeypatch.setattr(module, "get_dp_group", lambda: SimpleNamespace(cpu_group=None))
|
||||
return state
|
||||
|
||||
|
||||
def test_dp_ranks_replay_together(dp_state):
|
||||
"""A trimming rank makes the others replay too, on every rank's count."""
|
||||
short = _input_batch([100, 100], [100, 100], [True, True])
|
||||
dp_state.other = (False, 200)
|
||||
assert _prepare(dp_state, short, CUDAGraphMode.NONE) == (None, None)
|
||||
|
||||
dp_state.other = (True, 300) # rank 1 trims to 300 rows
|
||||
replay, build = _prepare(dp_state, short, CUDAGraphMode.NONE)
|
||||
assert replay is not None
|
||||
assert replay.rows.tolist() == list(range(200)) # this rank keeps every row
|
||||
dp_metadata = replay.forward_context.dp_metadata
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [200, 300]
|
||||
|
||||
|
||||
def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
|
||||
dummy = _input_batch([512], [512], [False])
|
||||
dp_state.other = (True, 129)
|
||||
replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE)
|
||||
assert replay is not None and replay.rows.shape[0] == WINDOW
|
||||
dp_metadata = replay.forward_context.dp_metadata
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129]
|
||||
@@ -260,7 +260,7 @@ def test_quantization(
|
||||
model,
|
||||
model_impl="auto",
|
||||
enforce_eager=True,
|
||||
**quantization_kwargs, # type: ignore[arg-type]
|
||||
**quantization_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
|
||||
@@ -270,7 +270,7 @@ def test_quantization(
|
||||
model,
|
||||
model_impl="transformers",
|
||||
enforce_eager=True,
|
||||
**quantization_kwargs, # type: ignore[arg-type]
|
||||
**quantization_kwargs,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.using_transformers_backend()
|
||||
|
||||
Reference in New Issue
Block a user