Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import create_new_process_for_each_test
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"Cascade attention backends FLASH_ATTN and FLASHINFER are notsupported on ROCm",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("attn_backend", ["FLASH_ATTN", "FLASHINFER"])
|
||||
def test_cascade_attention(example_system_message, attn_backend):
|
||||
prompt = "\n<User>: Implement fibonacci sequence in Python.\n<Claude>:"
|
||||
|
||||
if attn_backend == "FLASHINFER":
|
||||
pytest.skip(
|
||||
"This test is failing with FlashInfer backend and "
|
||||
"needs investigation. See issue #25679."
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen2-1.5B-Instruct", attention_config={"backend": attn_backend}
|
||||
)
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
|
||||
|
||||
# No cascade attention.
|
||||
single_prompt = [example_system_message + prompt]
|
||||
responses = llm.generate(single_prompt, sampling_params)
|
||||
ref_output = responses[0].outputs[0].text
|
||||
|
||||
# (Probably) Use cascade attention.
|
||||
prompts = [example_system_message + prompt] * 64
|
||||
responses = llm.generate(prompts, sampling_params)
|
||||
for response in responses:
|
||||
assert response.outputs[0].text == ref_output
|
||||
@@ -83,13 +83,20 @@ def _get_hisparse_worker(runner: VllmRunner) -> HiSparseConnectorWorker:
|
||||
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"with_offloading", [False, True], ids=["standalone", "offload"]
|
||||
"with_offloading,attention_backend",
|
||||
[
|
||||
pytest.param(False, None, id="standalone"),
|
||||
pytest.param(True, None, id="offload"),
|
||||
# Offloading does not touch the attention kernels: FA4 runs standalone.
|
||||
pytest.param(False, "FLASH_ATTN_MLA_SPARSE_FA4", id="standalone-fa4"),
|
||||
],
|
||||
)
|
||||
@fork_new_process_for_each_test
|
||||
def test_hisparse_spill_and_prefix_restore(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
vllm_runner: type[VllmRunner],
|
||||
with_offloading: bool,
|
||||
attention_backend: str | None,
|
||||
):
|
||||
"""Spilled prefixes restore and FULL-graph decode writes reach host KV.
|
||||
|
||||
@@ -101,6 +108,8 @@ def test_hisparse_spill_and_prefix_restore(
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.major < 9:
|
||||
pytest.skip("Sparse MLA requires Hopper or newer")
|
||||
if attention_backend and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("FA4 sparse MLA requires SM 10.x")
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
|
||||
@@ -142,6 +151,7 @@ def test_hisparse_spill_and_prefix_restore(
|
||||
)
|
||||
),
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
attention_backend=attention_backend,
|
||||
block_size=64,
|
||||
max_model_len=320,
|
||||
max_num_batched_tokens=1024,
|
||||
|
||||
@@ -74,6 +74,12 @@ def test_kv_sharing_fast_prefill(
|
||||
"ROCm: torch.compile produces incorrect output for gemma-3n's GELU "
|
||||
"with tanh approximation. Use enforce_eager=True instead."
|
||||
)
|
||||
if not enforce_eager and not use_v2_model_runner and current_platform.is_cuda():
|
||||
pytest.skip(
|
||||
"cublasCreate fails with CUBLAS_STATUS_ALLOC_FAILED during memory "
|
||||
"profiling with torch 2.14: "
|
||||
"https://github.com/pytorch/pytorch/issues/199944"
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
|
||||
compilation_config = CompilationConfig(
|
||||
|
||||
@@ -16,7 +16,10 @@ from tests.utils import create_new_process_for_each_test
|
||||
from vllm import LLM, SamplingParams, TokensPrompt
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -59,6 +62,10 @@ MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
|
||||
# where block boundaries fall. XPU rounds GDN block sizes up to a multiple of 64
|
||||
# (XPUPlatform.update_block_size_for_backend), so use one.
|
||||
BLOCK_SIZE = 576
|
||||
# The step actions also encode the split at the prompt's last block boundary,
|
||||
# which a GDN prefill backend with internal checkpoints (FlashInfer) skips; see
|
||||
# get_internal_checkpoint_step_configs.
|
||||
GDN_PREFILL_CONFIG = {"gdn_prefill_backend": "triton"}
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
NUM_HIDDEN_LAYERS = 1
|
||||
cur_step_action_idx = 0
|
||||
@@ -129,6 +136,10 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
first_token_id_index = (
|
||||
num_computed_tokens + 1
|
||||
) # bonus token isn't considered as computed
|
||||
# num_accepted_tokens_cpu is filled by a non-blocking D2H copy in the
|
||||
# same step; wait for it or a lagging GPU yields stale counts.
|
||||
if self.num_accepted_tokens_event is not None:
|
||||
self.num_accepted_tokens_event.synchronize()
|
||||
first_token_id_index += self.input_batch.num_accepted_tokens_cpu[0].item()
|
||||
proposed_draft_token_ids = [
|
||||
prompt_token_ids[
|
||||
@@ -878,6 +889,73 @@ def get_mamba_prefix_cache_step_configs(
|
||||
return tests
|
||||
|
||||
|
||||
def get_internal_checkpoint_step_configs() -> dict[str, TestConfig]:
|
||||
"""Async-scheduling steps when the GDN prefill exports the checkpoint at
|
||||
the prompt's last block boundary itself, so the final chunk runs whole.
|
||||
|
||||
The checkpoint goes in the second-to-last block the chunk reaches, keyed to
|
||||
the boundary: column 1 holds the state at B for a 2B + 10 prompt.
|
||||
"""
|
||||
B = BLOCK_SIZE
|
||||
return {
|
||||
"prompt_2_block_size": TestConfig(
|
||||
num_prompt_tokens=B * 2,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 2, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(B * 2, 4, [1, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
|
||||
],
|
||||
),
|
||||
"prompt_2_block_size_10": TestConfig(
|
||||
num_prompt_tokens=B * 2 + 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 2 + 10, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(B * 2 + 10, 4, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size": TestConfig(
|
||||
num_prompt_tokens=B * 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
B * 5,
|
||||
B * 5,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(4, 9),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(
|
||||
B * 10,
|
||||
4,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1],
|
||||
(9, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size_10": TestConfig(
|
||||
num_prompt_tokens=B * 10 + 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
B * 5,
|
||||
B * 5 + 10,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(4, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv1(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
):
|
||||
@@ -898,7 +976,7 @@ def _run_mamba_prefix_cache_mrv1(
|
||||
load_format="dummy",
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_cache_mode="align",
|
||||
additional_config=GDN_PREFILL_CONFIG,
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
@@ -1050,8 +1128,11 @@ def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
async_scheduling: bool,
|
||||
additional_config: dict[str, Any] = GDN_PREFILL_CONFIG,
|
||||
):
|
||||
internal_checkpoints = additional_config["gdn_prefill_backend"] == "flashinfer"
|
||||
global async_scheduling_mode
|
||||
async_scheduling_mode = async_scheduling
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
@@ -1191,8 +1272,24 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
with gpu_sync_allowed():
|
||||
start_pos = input_batch.positions[input_batch.query_start_loc[0]].item()
|
||||
assert cur_step_action.num_computed_tokens_start == start_pos
|
||||
if scheduler_output.scheduled_new_reqs and "snapshot_col" in captured:
|
||||
captured["snapshot"] = block_states(self, captured.pop("snapshot_col"))
|
||||
return ret
|
||||
|
||||
def block_states(runner: MRV2GPUModelRunner, col: int) -> list[list[torch.Tensor]]:
|
||||
# Each Mamba layer's (conv, temporal) states in block column col.
|
||||
kv_cache_config = captured["kv_cache_config"]
|
||||
forward_context = runner.vllm_config.compilation_config.static_forward_context
|
||||
return [
|
||||
[
|
||||
temporal_block(state, captured["block_tables"][group_id], col).clone()
|
||||
for state in forward_context[layer_name].kv_cache
|
||||
]
|
||||
for group_ids in get_mamba_groups(kv_cache_config).values()
|
||||
for group_id in group_ids
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names
|
||||
]
|
||||
|
||||
def fake_sample(
|
||||
self: MRV2GPUModelRunner,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1254,7 +1351,7 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
skip_tokenizer_init=True,
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_cache_mode="align",
|
||||
additional_config=additional_config,
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
@@ -1267,7 +1364,11 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
)
|
||||
|
||||
try:
|
||||
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
tests = (
|
||||
get_internal_checkpoint_step_configs()
|
||||
if internal_checkpoints
|
||||
else get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
)
|
||||
|
||||
global step_actions
|
||||
global cur_step_action_idx
|
||||
@@ -1313,6 +1414,35 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
f"got events={second_events!r}"
|
||||
)
|
||||
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
|
||||
|
||||
if internal_checkpoints:
|
||||
# The checkpoint a 2B + 10 prefill exports at B (column 1) is the
|
||||
# final state of a prefill that stops at B (column 0). Distinct
|
||||
# tokens make a shifted conv window visible.
|
||||
tokens = [(i * 7919) % 30000 + 1 for i in range(BLOCK_SIZE * 2 + 10)]
|
||||
scheduler = engine.llm_engine.engine_core.engine_core.scheduler
|
||||
states = []
|
||||
for num_tokens, col in ((BLOCK_SIZE, 0), (len(tokens), 1)):
|
||||
captured["snapshot_col"] = col
|
||||
_ = engine.generate(
|
||||
[TokensPrompt(prompt_token_ids=tokens[:num_tokens])],
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
states.append(captured.pop("snapshot"))
|
||||
assert scheduler.reset_prefix_cache()
|
||||
hf_config = engine.llm_engine.model_config.hf_text_config
|
||||
state_len = hf_config.linear_conv_kernel_dim - 1
|
||||
for (ref_conv, ref_ssm), (conv, ssm) in zip(*states):
|
||||
if not is_conv_state_dim_first():
|
||||
ref_conv, conv = ref_conv.mT, conv.mT
|
||||
# Under MTP the conv state has num_spec scratch columns after
|
||||
# the window.
|
||||
for new, ref in (
|
||||
(conv[..., :state_len], ref_conv[..., :state_len]),
|
||||
(ssm, ref_ssm),
|
||||
):
|
||||
rel_err = (new.float() - ref.float()).norm() / ref.float().norm()
|
||||
assert rel_err < 1e-2, rel_err
|
||||
finally:
|
||||
del engine
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1327,3 +1457,26 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or not (
|
||||
current_platform.is_device_capability(90)
|
||||
or (
|
||||
(
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
and current_platform.get_cuda_runtime_major() >= 13
|
||||
)
|
||||
),
|
||||
reason="FlashInfer GDN prefill needs SM90, or SM100/SM120 with CUDA 13.",
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch,
|
||||
async_scheduling=True,
|
||||
additional_config={"gdn_prefill_backend": "flashinfer"},
|
||||
)
|
||||
|
||||
@@ -8,23 +8,26 @@ request is queued before the first step, so the first prefill has the same
|
||||
shape in both runs. Give the engine its own process and the comparison turns
|
||||
into a coin toss: prefill starts before the last requests arrive over IPC, so
|
||||
one boot batches two requests where the other batches one. The batch shape
|
||||
sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties
|
||||
flip — measured at up to 5 of 16 completions, with sharding disabled on both
|
||||
sides. A real sharding bug — rows routed to the wrong request, misaligned
|
||||
gathers — produces many-nats divergence and scrambled top-k sets."""
|
||||
sets the GEMM reduction order and can make near-ties flip. A real sharding
|
||||
bug — rows routed to the wrong request, misaligned gathers — produces
|
||||
many-nats divergence and scrambled top-k sets."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.logprobs import Logprob, LogprobsOnePosition
|
||||
|
||||
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
GPT_OSS_MODEL = "trl-internal-testing/tiny-GptOssForCausalLM"
|
||||
GPT_OSS_REVISION = "e82685101216af1953707faa42567d232a5e4d45"
|
||||
|
||||
# Misrouted rows move logprobs by nats; anything under this is float noise.
|
||||
# With matched batches the two boots agreed bitwise over 320 positions, so the
|
||||
# bound is not expected to be reached.
|
||||
LOGPROB_TOL = 0.5
|
||||
|
||||
PROMPTS = [
|
||||
@@ -58,18 +61,52 @@ def _is_sharded_sampling_active(worker) -> bool:
|
||||
return worker.model_runner.batch_sharder is not None
|
||||
|
||||
|
||||
def _sampling_params() -> list[SamplingParams]:
|
||||
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))]
|
||||
def _local_logits_match_full(worker) -> dict[str, bool]:
|
||||
"""Reassemble rank-local logits and compare with the full path."""
|
||||
model = worker.model_runner.model
|
||||
hidden_size = model.config.hidden_size
|
||||
dtype = model.lm_head.weight.dtype
|
||||
hidden_states = torch.linspace(
|
||||
-1,
|
||||
1,
|
||||
steps=2 * hidden_size,
|
||||
device=worker.model_runner.device,
|
||||
dtype=dtype,
|
||||
).view(2, hidden_size)
|
||||
|
||||
full_logits = model.compute_logits(hidden_states)
|
||||
local_logits = model.compute_logits_local(hidden_states)
|
||||
assert local_logits.shape[-1] == model.lm_head.num_embeddings_per_partition
|
||||
gathered_logits = tensor_model_parallel_all_gather(local_logits)
|
||||
gathered_logits = gathered_logits[..., : model.config.vocab_size]
|
||||
|
||||
has_full_logits = full_logits is not None
|
||||
return {
|
||||
"has_full_logits": has_full_logits,
|
||||
"matches": not has_full_logits or torch.equal(full_logits, gathered_logits),
|
||||
}
|
||||
|
||||
|
||||
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
def _sampling_params(num_prompts: int) -> list[SamplingParams]:
|
||||
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(num_prompts)]
|
||||
|
||||
|
||||
def _generate(
|
||||
disable_sharding: bool,
|
||||
model: str,
|
||||
prompts: list[str],
|
||||
sampling_params: list[SamplingParams],
|
||||
revision: str | None,
|
||||
):
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
model=model,
|
||||
revision=revision,
|
||||
tokenizer_revision=revision,
|
||||
tensor_parallel_size=2,
|
||||
enable_batch_sharded_sampling=not disable_sharding,
|
||||
max_model_len=1024,
|
||||
# More requests than ranks, so round-robin slot ownership splits the
|
||||
# batch across both ranks.
|
||||
# Keep capacity at least as large as the TP size. The actual request
|
||||
# count may be smaller to exercise an empty rank.
|
||||
max_num_seqs=len(PROMPTS),
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=False,
|
||||
@@ -83,12 +120,14 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
# intended sampling mode.
|
||||
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
|
||||
assert all(mode == (not disable_sharding) for mode in modes), modes
|
||||
params = _sampling_params()
|
||||
# Two waves: the second reuses the request slots freed by the first
|
||||
# (in whatever order requests finished), covering the cross-rank
|
||||
# slot-recycling path that request ownership derives from.
|
||||
wave1 = llm.generate(PROMPTS, params)
|
||||
wave2 = llm.generate(PROMPTS[::-1], params[::-1])
|
||||
if not disable_sharding:
|
||||
logits_checks = llm.llm_engine.collective_rpc(_local_logits_match_full)
|
||||
assert any(check["has_full_logits"] for check in logits_checks)
|
||||
assert all(check["matches"] for check in logits_checks)
|
||||
# The second wave reuses request slots freed by the first,
|
||||
# covering the cross-rank slot-recycling path that ownership uses.
|
||||
wave1 = llm.generate(prompts, sampling_params)
|
||||
wave2 = llm.generate(prompts[::-1], sampling_params[::-1])
|
||||
return wave1 + wave2
|
||||
finally:
|
||||
del llm
|
||||
@@ -178,25 +217,11 @@ def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None):
|
||||
_assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
# Queue every request before the first step, so both boots schedule the
|
||||
# same batches -- see the module docstring.
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
ref_outputs = _generate(monkeypatch, disable_sharding=True)
|
||||
shard_outputs = _generate(monkeypatch, disable_sharding=False)
|
||||
|
||||
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
|
||||
params = _sampling_params()
|
||||
def _assert_outputs_match(ref_outputs, shard_outputs, params) -> None:
|
||||
assert len(ref_outputs) == len(shard_outputs) == len(params)
|
||||
divergent = []
|
||||
for i, (ref, out, sampling_params) in enumerate(
|
||||
zip(ref_outputs, shard_outputs, params + params[::-1])
|
||||
zip(ref_outputs, shard_outputs, params)
|
||||
):
|
||||
ref_completion = ref.outputs[0]
|
||||
out_completion = out.outputs[0]
|
||||
@@ -244,7 +269,60 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
)
|
||||
|
||||
assert not divergent, (
|
||||
f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at "
|
||||
f"{len(divergent)}/{len(params)} completions diverged at "
|
||||
f"(prompt, position) {divergent}; the sampling modes did not agree "
|
||||
"token for token with in-process scheduling"
|
||||
)
|
||||
|
||||
|
||||
def _configure_test_environment(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
# Queue every request before the first step, so both boots schedule the
|
||||
# same batches -- see the module docstring.
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
("model", "revision", "prompts"),
|
||||
[
|
||||
pytest.param(MODEL, None, PROMPTS, id="llama"),
|
||||
pytest.param(MODEL, None, PROMPTS[:1], id="llama-empty-rank"),
|
||||
pytest.param(
|
||||
GPT_OSS_MODEL,
|
||||
GPT_OSS_REVISION,
|
||||
PROMPTS,
|
||||
id="gpt-oss",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sharded_sampling_outputs_match(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
revision: str | None,
|
||||
prompts: list[str],
|
||||
):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback."""
|
||||
_configure_test_environment(monkeypatch)
|
||||
sampling_params = _sampling_params(len(prompts))
|
||||
|
||||
ref_outputs = _generate(
|
||||
disable_sharding=True,
|
||||
model=model,
|
||||
prompts=prompts,
|
||||
sampling_params=sampling_params,
|
||||
revision=revision,
|
||||
)
|
||||
shard_outputs = _generate(
|
||||
disable_sharding=False,
|
||||
model=model,
|
||||
prompts=prompts,
|
||||
sampling_params=sampling_params,
|
||||
revision=revision,
|
||||
)
|
||||
|
||||
expected_params = sampling_params + sampling_params[::-1]
|
||||
_assert_outputs_match(ref_outputs, shard_outputs, expected_params)
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
|
||||
from ...utils import compute_acceptance_rate
|
||||
|
||||
|
||||
def test_medusa_acceptance_rate(
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
|
||||
|
||||
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
|
||||
speculation path actually accepts tokens — unlike test_medusa_correctness,
|
||||
which uses a random head and only validates output correctness.
|
||||
"""
|
||||
target_model = "lmsys/vicuna-7b-v1.3"
|
||||
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
|
||||
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
|
||||
|
||||
with vllm_runner(
|
||||
target_model,
|
||||
block_size=None,
|
||||
trust_remote_code=False,
|
||||
speculative_config={
|
||||
"method": "medusa",
|
||||
"model": medusa_model,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=None,
|
||||
disable_log_stats=False,
|
||||
compilation_config=CompilationConfig(),
|
||||
) as spec_runner:
|
||||
spec_runner.llm.generate(prompts, sampling_config)
|
||||
metrics = spec_runner.llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
|
||||
min_acceptance_rate = 0.198
|
||||
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
|
||||
|
||||
# Regression guard at 90% of the measured baseline.
|
||||
assert acceptance_rate >= min_acceptance_rate, (
|
||||
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
|
||||
)
|
||||
@@ -18,17 +18,18 @@ from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling
|
||||
|
||||
# Even with in-process scheduling, near-tie tokens still flip on up to 3 of 20
|
||||
# prompts in CI, greedy and seeded. A sharding bug diverges wholesale instead.
|
||||
MAX_DIVERGENT_PROMPTS = 3
|
||||
|
||||
|
||||
def _is_sharded_sampling_active(worker) -> bool:
|
||||
return worker.model_runner.batch_sharder is not None
|
||||
|
||||
|
||||
def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Batch-sharded sampling must be a bit-exact drop-in for replicated
|
||||
sampling under MTP spec decoding: the collectives move the same logits
|
||||
bytes, Gumbel keys derive from (request slot, position, seed), and slot
|
||||
assignment is rank-deterministic. Both runs here are spec decode with
|
||||
identical math, so outputs must match exactly."""
|
||||
"""Batch-sharded sampling must match replicated sampling under MTP spec
|
||||
decoding, up to measured near-tie noise."""
|
||||
tp_size = 2
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
model_name = "Qwen/Qwen3.5-0.8B-Base"
|
||||
@@ -96,8 +97,7 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
f" replicated: {ref.outputs[0].text!r}\n"
|
||||
f" sharded: {out.outputs[0].text!r}"
|
||||
)
|
||||
assert not divergent, (
|
||||
assert len(divergent) <= MAX_DIVERGENT_PROMPTS, (
|
||||
f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged "
|
||||
f"at {divergent}; the sampling modes did not agree token for "
|
||||
"token with in-process scheduling"
|
||||
f"at {divergent}, beyond near-tie noise"
|
||||
)
|
||||
|
||||
@@ -88,7 +88,6 @@ def test_mtp_speculative_mixed_batch_short_prefill(
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=enable_prefix_caching,
|
||||
mamba_cache_mode="align" if enable_prefix_caching else "none",
|
||||
) as llm:
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
@@ -161,7 +160,6 @@ def test_qwen_mtp_mamba_prefix_cache_hit_is_bounded(vllm_runner, monkeypatch):
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 2},
|
||||
) as runner:
|
||||
llm = runner.get_llm()
|
||||
|
||||
@@ -60,7 +60,6 @@ def _check_replayssm_parity(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
mamba_backend=mamba_backend,
|
||||
)
|
||||
@@ -122,32 +121,48 @@ def test_replayssm_flashinfer_decode_matches_baseline(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
|
||||
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_replayssm_flashinfer_spec_decode_matches_baseline(
|
||||
vllm_runner, model_name, monkeypatch, use_v2_model_runner
|
||||
):
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
speculative_config={
|
||||
"method": "ngram",
|
||||
"num_speculative_tokens": 3,
|
||||
"prompt_lookup_max": 3,
|
||||
},
|
||||
)
|
||||
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
|
||||
baseline = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
mamba_backend="flashinfer",
|
||||
**common,
|
||||
) as llm:
|
||||
replay = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
envs.disable_envs_cache()
|
||||
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
|
||||
assert (
|
||||
llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
baseline = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
mamba_backend="flashinfer",
|
||||
**common,
|
||||
) as llm:
|
||||
assert (
|
||||
llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
replay = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
finally:
|
||||
envs.disable_envs_cache()
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline,
|
||||
@@ -165,7 +180,6 @@ def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner)
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
mamba_backend="flashinfer",
|
||||
tensor_parallel_size=2,
|
||||
disable_log_stats=False,
|
||||
@@ -233,7 +247,6 @@ def _check_replayssm_prefix_caching_parity(
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=True,
|
||||
mamba_cache_mode="align",
|
||||
disable_log_stats=False, # required for llm.get_metrics()
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user