Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -5,7 +5,13 @@ import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import (
|
||||
assert_request_outputs_match,
|
||||
compute_acceptance_len,
|
||||
get_test_prompts,
|
||||
)
|
||||
from .._correctness import check_mtp_correctness
|
||||
|
||||
|
||||
@@ -37,3 +43,107 @@ def test_mtp_correctness(
|
||||
expected_accuracy_threshold,
|
||||
vllm_runner,
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_mtp_draft_lm_head_quantization(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""A quantized draft copy of the shared lm_head only changes drafting:
|
||||
greedy outputs match stock MTP and acceptance stays within 10%."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
prompts = get_test_prompts(mm_enabled=False)
|
||||
results = {}
|
||||
# NVFP4 needs a CUDA W4A16 kernel.
|
||||
quants = (None, "fp8", "nvfp4") if current_platform.is_cuda() else (None, "fp8")
|
||||
for quant in quants:
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
block_size=None,
|
||||
enable_chunked_prefill=None,
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
"draft_lm_head_quantization": quant,
|
||||
},
|
||||
disable_log_stats=False,
|
||||
) as runner:
|
||||
outputs = runner.llm.chat(prompts, sampling_config)
|
||||
results[quant] = (outputs, compute_acceptance_len(runner.llm.get_metrics()))
|
||||
|
||||
ref_outputs, ref_al = results.pop(None)
|
||||
for quant, (outputs, al) in results.items():
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
# Not bit-exact: a different acceptance length changes the target's
|
||||
# verify batch shapes, which can flip a near-tied greedy token.
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context=f"draft_lm_head_quantization={quant}",
|
||||
)
|
||||
assert al >= 0.9 * ref_al, (
|
||||
f"{quant}: acceptance length {al:.3f} vs {ref_al:.3f}"
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
|
||||
)
|
||||
def test_mtp_draft_lm_head_quantization_survives_sleep_and_reload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""The quantized draft head is derived from the target head: it must come
|
||||
back intact after a level 1 sleep and be re-derived after a target weight
|
||||
reload, so outputs and acceptance match the run before."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
prompts = get_test_prompts(mm_enabled=False)
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
block_size=None,
|
||||
enable_chunked_prefill=None,
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
"draft_lm_head_quantization": "nvfp4",
|
||||
},
|
||||
enable_sleep_mode=True,
|
||||
disable_log_stats=False,
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
|
||||
def run():
|
||||
before = llm.get_metrics()
|
||||
outputs = llm.chat(prompts, sampling_config)
|
||||
return outputs, compute_acceptance_len(llm.get_metrics(), before)
|
||||
|
||||
ref_outputs, ref_al = run()
|
||||
|
||||
llm.sleep(level=1)
|
||||
llm.wake_up()
|
||||
outputs, al = run()
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context="after level 1 sleep",
|
||||
)
|
||||
assert al == pytest.approx(ref_al, rel=0.03)
|
||||
|
||||
llm.collective_rpc("reload_weights")
|
||||
outputs, al = run()
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context="after weight reload",
|
||||
)
|
||||
assert al == pytest.approx(ref_al, rel=0.03)
|
||||
|
||||
Reference in New Issue
Block a user