Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
from ....utils import create_new_process_for_each_test
if current_platform.is_rocm():
pytest.skip(
"Cascade attention backends FLASH_ATTN and FLASHINFER are notsupported on ROCm",
allow_module_level=True,
)
@create_new_process_for_each_test()
@pytest.mark.parametrize("attn_backend", ["FLASH_ATTN", "FLASHINFER"])
def test_cascade_attention(example_system_message, attn_backend):
prompt = "\n<User>: Implement fibonacci sequence in Python.\n<Claude>:"
if attn_backend == "FLASHINFER":
pytest.skip(
"This test is failing with FlashInfer backend and "
"needs investigation. See issue #25679."
)
llm = LLM(
model="Qwen/Qwen2-1.5B-Instruct", attention_config={"backend": attn_backend}
)
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
# No cascade attention.
single_prompt = [example_system_message + prompt]
responses = llm.generate(single_prompt, sampling_params)
ref_output = responses[0].outputs[0].text
# (Probably) Use cascade attention.
prompts = [example_system_message + prompt] * 64
responses = llm.generate(prompts, sampling_params)
for response in responses:
assert response.outputs[0].text == ref_output
+11 -1
View File
@@ -83,13 +83,20 @@ def _get_hisparse_worker(runner: VllmRunner) -> HiSparseConnectorWorker:
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
)
@pytest.mark.parametrize(
"with_offloading", [False, True], ids=["standalone", "offload"]
"with_offloading,attention_backend",
[
pytest.param(False, None, id="standalone"),
pytest.param(True, None, id="offload"),
# Offloading does not touch the attention kernels: FA4 runs standalone.
pytest.param(False, "FLASH_ATTN_MLA_SPARSE_FA4", id="standalone-fa4"),
],
)
@fork_new_process_for_each_test
def test_hisparse_spill_and_prefix_restore(
monkeypatch: pytest.MonkeyPatch,
vllm_runner: type[VllmRunner],
with_offloading: bool,
attention_backend: str | None,
):
"""Spilled prefixes restore and FULL-graph decode writes reach host KV.
@@ -101,6 +108,8 @@ def test_hisparse_spill_and_prefix_restore(
capability = current_platform.get_device_capability()
if capability is None or capability.major < 9:
pytest.skip("Sparse MLA requires Hopper or newer")
if attention_backend and not current_platform.is_device_capability_family(100):
pytest.skip("FA4 sparse MLA requires SM 10.x")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
@@ -142,6 +151,7 @@ def test_hisparse_spill_and_prefix_restore(
)
),
kv_transfer_config=kv_transfer_config,
attention_backend=attention_backend,
block_size=64,
max_model_len=320,
max_num_batched_tokens=1024,
@@ -74,6 +74,12 @@ def test_kv_sharing_fast_prefill(
"ROCm: torch.compile produces incorrect output for gemma-3n's GELU "
"with tanh approximation. Use enforce_eager=True instead."
)
if not enforce_eager and not use_v2_model_runner and current_platform.is_cuda():
pytest.skip(
"cublasCreate fails with CUBLAS_STATUS_ALLOC_FAILED during memory "
"profiling with torch 2.14: "
"https://github.com/pytorch/pytorch/issues/199944"
)
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
compilation_config = CompilationConfig(
+158 -5
View File
@@ -16,7 +16,10 @@ from tests.utils import create_new_process_for_each_test
from vllm import LLM, SamplingParams, TokensPrompt
from vllm.config import CacheConfig
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
is_conv_state_dim_first,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -59,6 +62,10 @@ MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
# where block boundaries fall. XPU rounds GDN block sizes up to a multiple of 64
# (XPUPlatform.update_block_size_for_backend), so use one.
BLOCK_SIZE = 576
# The step actions also encode the split at the prompt's last block boundary,
# which a GDN prefill backend with internal checkpoints (FlashInfer) skips; see
# get_internal_checkpoint_step_configs.
GDN_PREFILL_CONFIG = {"gdn_prefill_backend": "triton"}
DEVICE_TYPE = current_platform.device_type
NUM_HIDDEN_LAYERS = 1
cur_step_action_idx = 0
@@ -129,6 +136,10 @@ def get_fake_propose_draft_token_ids_fn():
first_token_id_index = (
num_computed_tokens + 1
) # bonus token isn't considered as computed
# num_accepted_tokens_cpu is filled by a non-blocking D2H copy in the
# same step; wait for it or a lagging GPU yields stale counts.
if self.num_accepted_tokens_event is not None:
self.num_accepted_tokens_event.synchronize()
first_token_id_index += self.input_batch.num_accepted_tokens_cpu[0].item()
proposed_draft_token_ids = [
prompt_token_ids[
@@ -878,6 +889,73 @@ def get_mamba_prefix_cache_step_configs(
return tests
def get_internal_checkpoint_step_configs() -> dict[str, TestConfig]:
"""Async-scheduling steps when the GDN prefill exports the checkpoint at
the prompt's last block boundary itself, so the final chunk runs whole.
The checkpoint goes in the second-to-last block the chunk reaches, keyed to
the boundary: column 1 holds the state at B for a 2B + 10 prompt.
"""
B = BLOCK_SIZE
return {
"prompt_2_block_size": TestConfig(
num_prompt_tokens=B * 2,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 2, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(B * 2, 4, [1, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
],
),
"prompt_2_block_size_10": TestConfig(
num_prompt_tokens=B * 2 + 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 2 + 10, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(B * 2 + 10, 4, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
"prompt_10_block_size": TestConfig(
num_prompt_tokens=B * 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
B * 5,
B * 5,
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1],
(4, 9),
(-1, -1),
),
StepAction(
B * 10,
4,
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1],
(9, 10),
(-1, -1),
),
],
),
"prompt_10_block_size_10": TestConfig(
num_prompt_tokens=B * 10 + 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
B * 5,
B * 5 + 10,
[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1],
(4, 10),
(-1, -1),
),
],
),
}
def _run_mamba_prefix_cache_mrv1(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
):
@@ -898,7 +976,7 @@ def _run_mamba_prefix_cache_mrv1(
load_format="dummy",
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
additional_config=GDN_PREFILL_CONFIG,
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -1050,8 +1128,11 @@ def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
def _run_mamba_prefix_cache_mrv2(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
monkeypatch: pytest.MonkeyPatch,
async_scheduling: bool,
additional_config: dict[str, Any] = GDN_PREFILL_CONFIG,
):
internal_checkpoints = additional_config["gdn_prefill_backend"] == "flashinfer"
global async_scheduling_mode
async_scheduling_mode = async_scheduling
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@@ -1191,8 +1272,24 @@ def _run_mamba_prefix_cache_mrv2(
with gpu_sync_allowed():
start_pos = input_batch.positions[input_batch.query_start_loc[0]].item()
assert cur_step_action.num_computed_tokens_start == start_pos
if scheduler_output.scheduled_new_reqs and "snapshot_col" in captured:
captured["snapshot"] = block_states(self, captured.pop("snapshot_col"))
return ret
def block_states(runner: MRV2GPUModelRunner, col: int) -> list[list[torch.Tensor]]:
# Each Mamba layer's (conv, temporal) states in block column col.
kv_cache_config = captured["kv_cache_config"]
forward_context = runner.vllm_config.compilation_config.static_forward_context
return [
[
temporal_block(state, captured["block_tables"][group_id], col).clone()
for state in forward_context[layer_name].kv_cache
]
for group_ids in get_mamba_groups(kv_cache_config).values()
for group_id in group_ids
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names
]
def fake_sample(
self: MRV2GPUModelRunner,
hidden_states: torch.Tensor,
@@ -1254,7 +1351,7 @@ def _run_mamba_prefix_cache_mrv2(
skip_tokenizer_init=True,
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
additional_config=additional_config,
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -1267,7 +1364,11 @@ def _run_mamba_prefix_cache_mrv2(
)
try:
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
tests = (
get_internal_checkpoint_step_configs()
if internal_checkpoints
else get_mamba_prefix_cache_step_configs(async_scheduling)
)
global step_actions
global cur_step_action_idx
@@ -1313,6 +1414,35 @@ def _run_mamba_prefix_cache_mrv2(
f"got events={second_events!r}"
)
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
if internal_checkpoints:
# The checkpoint a 2B + 10 prefill exports at B (column 1) is the
# final state of a prefill that stops at B (column 0). Distinct
# tokens make a shifted conv window visible.
tokens = [(i * 7919) % 30000 + 1 for i in range(BLOCK_SIZE * 2 + 10)]
scheduler = engine.llm_engine.engine_core.engine_core.scheduler
states = []
for num_tokens, col in ((BLOCK_SIZE, 0), (len(tokens), 1)):
captured["snapshot_col"] = col
_ = engine.generate(
[TokensPrompt(prompt_token_ids=tokens[:num_tokens])],
sampling_params=sampling_params,
)
states.append(captured.pop("snapshot"))
assert scheduler.reset_prefix_cache()
hf_config = engine.llm_engine.model_config.hf_text_config
state_len = hf_config.linear_conv_kernel_dim - 1
for (ref_conv, ref_ssm), (conv, ssm) in zip(*states):
if not is_conv_state_dim_first():
ref_conv, conv = ref_conv.mT, conv.mT
# Under MTP the conv state has num_spec scratch columns after
# the window.
for new, ref in (
(conv[..., :state_len], ref_conv[..., :state_len]),
(ssm, ref_ssm),
):
rel_err = (new.float() - ref.float()).norm() / ref.float().norm()
assert rel_err < 1e-2, rel_err
finally:
del engine
torch.accelerator.empty_cache()
@@ -1327,3 +1457,26 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
@pytest.mark.skipif(
not current_platform.is_cuda()
or not (
current_platform.is_device_capability(90)
or (
(
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
)
and current_platform.get_cuda_runtime_major() >= 13
)
),
reason="FlashInfer GDN prefill needs SM90, or SM100/SM120 with CUDA 13.",
)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2_async_flashinfer(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(
monkeypatch,
async_scheduling=True,
additional_config={"gdn_prefill_backend": "flashinfer"},
)
+115 -37
View File
@@ -8,23 +8,26 @@ request is queued before the first step, so the first prefill has the same
shape in both runs. Give the engine its own process and the comparison turns
into a coin toss: prefill starts before the last requests arrive over IPC, so
one boot batches two requests where the other batches one. The batch shape
sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties
flip — measured at up to 5 of 16 completions, with sharding disabled on both
sides. A real sharding bug — rows routed to the wrong request, misaligned
gathers — produces many-nats divergence and scrambled top-k sets."""
sets the GEMM reduction order and can make near-ties flip. A real sharding
bug — rows routed to the wrong request, misaligned gathers — produces
many-nats divergence and scrambled top-k sets."""
import pytest
import torch
from tests.utils import multi_gpu_test
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed import (
cleanup_dist_env_and_memory,
tensor_model_parallel_all_gather,
)
from vllm.logprobs import Logprob, LogprobsOnePosition
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
GPT_OSS_MODEL = "trl-internal-testing/tiny-GptOssForCausalLM"
GPT_OSS_REVISION = "e82685101216af1953707faa42567d232a5e4d45"
# Misrouted rows move logprobs by nats; anything under this is float noise.
# With matched batches the two boots agreed bitwise over 320 positions, so the
# bound is not expected to be reached.
LOGPROB_TOL = 0.5
PROMPTS = [
@@ -58,18 +61,52 @@ def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def _sampling_params() -> list[SamplingParams]:
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))]
def _local_logits_match_full(worker) -> dict[str, bool]:
"""Reassemble rank-local logits and compare with the full path."""
model = worker.model_runner.model
hidden_size = model.config.hidden_size
dtype = model.lm_head.weight.dtype
hidden_states = torch.linspace(
-1,
1,
steps=2 * hidden_size,
device=worker.model_runner.device,
dtype=dtype,
).view(2, hidden_size)
full_logits = model.compute_logits(hidden_states)
local_logits = model.compute_logits_local(hidden_states)
assert local_logits.shape[-1] == model.lm_head.num_embeddings_per_partition
gathered_logits = tensor_model_parallel_all_gather(local_logits)
gathered_logits = gathered_logits[..., : model.config.vocab_size]
has_full_logits = full_logits is not None
return {
"has_full_logits": has_full_logits,
"matches": not has_full_logits or torch.equal(full_logits, gathered_logits),
}
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
def _sampling_params(num_prompts: int) -> list[SamplingParams]:
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(num_prompts)]
def _generate(
disable_sharding: bool,
model: str,
prompts: list[str],
sampling_params: list[SamplingParams],
revision: str | None,
):
llm = LLM(
model=MODEL,
model=model,
revision=revision,
tokenizer_revision=revision,
tensor_parallel_size=2,
enable_batch_sharded_sampling=not disable_sharding,
max_model_len=1024,
# More requests than ranks, so round-robin slot ownership splits the
# batch across both ranks.
# Keep capacity at least as large as the TP size. The actual request
# count may be smaller to exercise an empty rank.
max_num_seqs=len(PROMPTS),
enforce_eager=True,
enable_prefix_caching=False,
@@ -83,12 +120,14 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
# intended sampling mode.
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
assert all(mode == (not disable_sharding) for mode in modes), modes
params = _sampling_params()
# Two waves: the second reuses the request slots freed by the first
# (in whatever order requests finished), covering the cross-rank
# slot-recycling path that request ownership derives from.
wave1 = llm.generate(PROMPTS, params)
wave2 = llm.generate(PROMPTS[::-1], params[::-1])
if not disable_sharding:
logits_checks = llm.llm_engine.collective_rpc(_local_logits_match_full)
assert any(check["has_full_logits"] for check in logits_checks)
assert all(check["matches"] for check in logits_checks)
# The second wave reuses request slots freed by the first,
# covering the cross-rank slot-recycling path that ownership uses.
wave1 = llm.generate(prompts, sampling_params)
wave2 = llm.generate(prompts[::-1], sampling_params[::-1])
return wave1 + wave2
finally:
del llm
@@ -178,25 +217,11 @@ def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None):
_assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1)
@multi_gpu_test(num_gpus=2)
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
# Queue every request before the first step, so both boots schedule the
# same batches -- see the module docstring.
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
ref_outputs = _generate(monkeypatch, disable_sharding=True)
shard_outputs = _generate(monkeypatch, disable_sharding=False)
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
params = _sampling_params()
def _assert_outputs_match(ref_outputs, shard_outputs, params) -> None:
assert len(ref_outputs) == len(shard_outputs) == len(params)
divergent = []
for i, (ref, out, sampling_params) in enumerate(
zip(ref_outputs, shard_outputs, params + params[::-1])
zip(ref_outputs, shard_outputs, params)
):
ref_completion = ref.outputs[0]
out_completion = out.outputs[0]
@@ -244,7 +269,60 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
)
assert not divergent, (
f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at "
f"{len(divergent)}/{len(params)} completions diverged at "
f"(prompt, position) {divergent}; the sampling modes did not agree "
"token for token with in-process scheduling"
)
def _configure_test_environment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
# Queue every request before the first step, so both boots schedule the
# same batches -- see the module docstring.
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
("model", "revision", "prompts"),
[
pytest.param(MODEL, None, PROMPTS, id="llama"),
pytest.param(MODEL, None, PROMPTS[:1], id="llama-empty-rank"),
pytest.param(
GPT_OSS_MODEL,
GPT_OSS_REVISION,
PROMPTS,
id="gpt-oss",
),
],
)
def test_sharded_sampling_outputs_match(
monkeypatch: pytest.MonkeyPatch,
model: str,
revision: str | None,
prompts: list[str],
):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback."""
_configure_test_environment(monkeypatch)
sampling_params = _sampling_params(len(prompts))
ref_outputs = _generate(
disable_sharding=True,
model=model,
prompts=prompts,
sampling_params=sampling_params,
revision=revision,
)
shard_outputs = _generate(
disable_sharding=False,
model=model,
prompts=prompts,
sampling_params=sampling_params,
revision=revision,
)
expected_params = sampling_params + sampling_params[::-1]
_assert_outputs_match(ref_outputs, shard_outputs, expected_params)
@@ -1,50 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts
from vllm import SamplingParams
from vllm.config import CompilationConfig
from ...utils import compute_acceptance_rate
def test_medusa_acceptance_rate(
sampling_config: SamplingParams,
vllm_runner,
):
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
speculation path actually accepts tokens — unlike test_medusa_correctness,
which uses a random head and only validates output correctness.
"""
target_model = "lmsys/vicuna-7b-v1.3"
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
with vllm_runner(
target_model,
block_size=None,
trust_remote_code=False,
speculative_config={
"method": "medusa",
"model": medusa_model,
"num_speculative_tokens": 3,
},
max_model_len=1024,
enforce_eager=True,
enable_chunked_prefill=None,
disable_log_stats=False,
compilation_config=CompilationConfig(),
) as spec_runner:
spec_runner.llm.generate(prompts, sampling_config)
metrics = spec_runner.llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
min_acceptance_rate = 0.198
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
# Regression guard at 90% of the measured baseline.
assert acceptance_rate >= min_acceptance_rate, (
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
)
@@ -18,17 +18,18 @@ from vllm.distributed import cleanup_dist_env_and_memory
from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling
# Even with in-process scheduling, near-tie tokens still flip on up to 3 of 20
# prompts in CI, greedy and seeded. A sharding bug diverges wholesale instead.
MAX_DIVERGENT_PROMPTS = 3
def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
"""Batch-sharded sampling must be a bit-exact drop-in for replicated
sampling under MTP spec decoding: the collectives move the same logits
bytes, Gumbel keys derive from (request slot, position, seed), and slot
assignment is rank-deterministic. Both runs here are spec decode with
identical math, so outputs must match exactly."""
"""Batch-sharded sampling must match replicated sampling under MTP spec
decoding, up to measured near-tie noise."""
tp_size = 2
_skip_if_insufficient_gpus_for_tp(tp_size)
model_name = "Qwen/Qwen3.5-0.8B-Base"
@@ -96,8 +97,7 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
f" replicated: {ref.outputs[0].text!r}\n"
f" sharded: {out.outputs[0].text!r}"
)
assert not divergent, (
assert len(divergent) <= MAX_DIVERGENT_PROMPTS, (
f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged "
f"at {divergent}; the sampling modes did not agree token for "
"token with in-process scheduling"
f"at {divergent}, beyond near-tie noise"
)
@@ -88,7 +88,6 @@ def test_mtp_speculative_mixed_batch_short_prefill(
trust_remote_code=True,
enable_chunked_prefill=True,
enable_prefix_caching=enable_prefix_caching,
mamba_cache_mode="align" if enable_prefix_caching else "none",
) as llm:
sampling_params = SamplingParams(
temperature=0.0,
@@ -161,7 +160,6 @@ def test_qwen_mtp_mamba_prefix_cache_hit_is_bounded(vllm_runner, monkeypatch):
enforce_eager=True,
enable_chunked_prefill=True,
enable_prefix_caching=True,
mamba_cache_mode="align",
speculative_config={"method": "mtp", "num_speculative_tokens": 2},
) as runner:
llm = runner.get_llm()
+32 -19
View File
@@ -60,7 +60,6 @@ def _check_replayssm_parity(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
tensor_parallel_size=tensor_parallel_size,
mamba_backend=mamba_backend,
)
@@ -122,32 +121,48 @@ def test_replayssm_flashinfer_decode_matches_baseline(
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_replayssm_flashinfer_spec_decode_matches_baseline(
vllm_runner, model_name, monkeypatch, use_v2_model_runner
):
common = dict(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
speculative_config={
"method": "ngram",
"num_speculative_tokens": 3,
"prompt_lookup_max": 3,
},
)
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
baseline = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
with vllm_runner(
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
mamba_backend="flashinfer",
**common,
) as llm:
replay = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
assert (
llm.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
baseline = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
with vllm_runner(
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
mamba_backend="flashinfer",
**common,
) as llm:
assert (
llm.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
replay = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
finally:
envs.disable_envs_cache()
check_logprobs_close(
outputs_0_lst=baseline,
@@ -165,7 +180,6 @@ def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner)
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
mamba_backend="flashinfer",
tensor_parallel_size=2,
disable_log_stats=False,
@@ -233,7 +247,6 @@ def _check_replayssm_prefix_caching_parity(
trust_remote_code=True,
enable_prefix_caching=True,
enable_chunked_prefill=True,
mamba_cache_mode="align",
disable_log_stats=False, # required for llm.get_metrics()
tensor_parallel_size=tensor_parallel_size,
)