diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index 769e3d4adc..b50a082727 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -52,6 +52,52 @@ steps: tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp" parallelism: 2 +# Speculative decoding under a CPU platform. The unit tests already pass on +# CPU but no CPU job runs them; the e2e tests gate the engine path itself, +# which unit tests cannot reach. +- label: CPU-Spec Decode Tests + key: cpu-spec-decode-tests + depends_on: [] + soft_fail: true + device: intel_cpu + no_plugin: true + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/cpu/ + - vllm/v1/worker/cpu_model_runner.py + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/attention/backends/cpu_attn.py + - vllm/config/speculative.py + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/models/qwen3_5_mtp.py + - vllm/model_executor/models/llama_eagle3.py + - vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py + - vllm/model_executor/layers/mamba/mamba_mixer2.py + - vllm/model_executor/layers/mamba/ops/cpu/ + - tests/v1/e2e/test_cpu_spec_decode.py + - tests/v1/spec_decode/ + - tests/v1/worker/ + commands: + - | + bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " + pytest -x -v -s \ + tests/v1/worker/test_gpu_autoregressive_speculator.py \ + tests/v1/worker/test_gpu_extract_hidden_states_speculator.py \ + tests/v1/worker/test_gpu_batch_ordering.py \ + tests/v1/worker/test_gpu_rejection_sampler_chunking.py \ + tests/v1/worker/test_eagle3_aux_hidden_states_pp.py \ + tests/v1/worker/test_spec_decode_embed_sharing_pp.py \ + tests/v1/spec_decode/test_adaptive_verification.py \ + tests/v1/spec_decode/test_dflash2.py \ + tests/v1/spec_decode/test_dflash_causality.py \ + tests/v1/spec_decode/test_eagle_draft_attn_metadata.py \ + tests/v1/spec_decode/test_draft_attention_backend_override.py \ + tests/v1/spec_decode/test_draft_moe_backend_override.py \ + tests/v1/spec_decode/test_ngram.py \ + tests/v1/spec_decode/test_llm_base_proposer_sampling.py + unset VLLM_CPU_CI_ENV + pytest -x -v -s tests/v1/e2e/test_cpu_spec_decode.py" + # Note: SDE can't be downloaded from CI host because of AWS WAF # - label: CPU-Compatibility Tests # depends_on: [] diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test.sh b/.buildkite/scripts/hardware_ci/run-cpu-test.sh index e47cc22c8f..74c7749d50 100644 --- a/.buildkite/scripts/hardware_ci/run-cpu-test.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test.sh @@ -83,8 +83,11 @@ rm -f "$build_log" # Run the image, setting --shm-size=4g for tensor parallel. Default to # HF_HUB_OFFLINE so a warm ~/.cache/huggingface doesn't hit the network; -# retry once online if the cache is missing something. -OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)' +# retry once online if the cache is missing something. vllm's get_config() +# wraps the raw huggingface_hub offline-mode errors in a generic ValueError +# (see transformers_utils/config.py), so match that message too or the +# fallback never triggers for a config-lookup cache miss. +OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified' run_test() { local hf_offline=$1 docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \ diff --git a/docs/features/README.md b/docs/features/README.md index 28362f4014..1df8960ee3 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -64,7 +64,7 @@ th:not(:first-child) { | [CP](../configuration/optimization.md#chunked-prefill) | [❌](https://github.com/vllm-project/vllm/issues/2729) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | [APC](automatic_prefix_caching.md) | [❌](https://github.com/vllm-project/vllm/issues/3687) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | [LoRA](lora.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [SD](speculative_decoding/README.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | +| [SD](speculative_decoding/README.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | CUDA graph | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | [❌](https://github.com/vllm-project/vllm/issues/26970) | | [pooling](../models/pooling_models/README.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | enc-dec | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | diff --git a/tests/v1/e2e/test_cpu_spec_decode.py b/tests/v1/e2e/test_cpu_spec_decode.py new file mode 100644 index 0000000000..2d34c11656 --- /dev/null +++ b/tests/v1/e2e/test_cpu_spec_decode.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU speculative-decoding end-to-end correctness.""" + +import os + +import pytest + +from tests.models.utils import check_logprobs_close +from tests.v1.e2e.spec_decode.utils import compute_acceptance_len +from vllm import LLM, SamplingParams +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON + +pytestmark = pytest.mark.cpu_model + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + +# Bound the KV cache so the run does not scale with host memory. +os.environ.setdefault("VLLM_CPU_KVCACHE_SPACE", "1") + +MODEL = "Qwen/Qwen3-0.6B" +NUM_LOGPROBS = 5 +SP = SamplingParams(max_tokens=48, temperature=0, logprobs=NUM_LOGPROBS) + +# Repetitive prompts so the ngram proposer finds matches and actually drafts; +# without hits the run is indistinguishable from plain decoding. +PROMPTS = [ + "The capital of France is Paris. The capital of Italy is Rome. " + "The capital of Spain is Madrid. The capital of France is", + "def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - b\n\n" + "def mul(a, b):\n return", + "one two three four five one two three four five one two three four", +] + + +def _make_llm(**overrides) -> LLM: + base = dict( + model=MODEL, + dtype="bfloat16", + max_model_len=1024, + disable_log_stats=False, + ) + base.update(overrides) + return LLM(**base) + + +def _tuples(outputs) -> list[tuple[list[int], str, object]]: + return [ + (list(o.outputs[0].token_ids), o.outputs[0].text, o.outputs[0].logprobs) + for o in outputs + ] + + +@pytest.fixture(scope="module") +def baseline_refs(): + llm = _make_llm() + refs = _tuples(llm.generate(PROMPTS, SP)) + del llm + return refs + + +def _spec_metric(llm: LLM, name: str) -> float: + for metric in llm.get_metrics(): + if metric.name == name: + return float(getattr(metric, "value", 0.0)) + return 0.0 + + +def test_ngram_spec_decode_matches_baseline(baseline_refs): + """Greedy output must be unchanged by speculative decoding. + + The drafted-token check keeps the test from passing vacuously when the + proposer silently produces nothing. + """ + llm = _make_llm( + speculative_config={ + "method": "ngram", + "prompt_lookup_max": 5, + "prompt_lookup_min": 3, + "num_speculative_tokens": 3, + }, + ) + got = _tuples(llm.generate(PROMPTS, SP)) + drafted = _spec_metric(llm, "vllm:spec_decode_num_draft_tokens") + accepted = _spec_metric(llm, "vllm:spec_decode_num_accepted_tokens") + del llm + + assert drafted > 0, "ngram proposer drafted no tokens; the test would be vacuous" + assert accepted > 0, f"no draft tokens accepted out of {drafted} drafted" + + check_logprobs_close( + outputs_0_lst=baseline_refs, + outputs_1_lst=got, + name_0="no_spec_decode", + name_1="ngram_spec_decode", + ) + + +SMOKE_PROMPTS = [ + "The capital of France is", + "2 + 2 equals", + "In one word, the color of the sky is", + "Q: If a train travels 60 miles in 1.5 hours, what is its average speed?\nA:", +] +SMOKE_SP = SamplingParams(temperature=0.0, max_tokens=32, ignore_eos=True) + +# Methods routed to the V2 runner, using the same target/draft pairs as the GPU +# suite. MTP needs no separate draft checkpoint: its speculator layer is +# trained jointly with, and shipped inside, the target model's own weights. +SMOKE_CONFIGS = [ + pytest.param( + "meta-llama/Llama-3.2-1B-Instruct", + { + "method": "eagle3", + "model": "nm-testing/Llama3_2_1B_speculator.eagle3", + "num_speculative_tokens": 3, + }, + id="eagle3", + ), + pytest.param( + "Qwen/Qwen3-4B", + { + "method": "dflash", + "model": "z-lab/Qwen3-4B-DFlash-b16", + # The checkpoint allows 16, but accepted tokens plateau around 64 + # regardless, so a larger draft only burns CPU on rejected tokens. + "num_speculative_tokens": 4, + }, + id="dflash", + ), + pytest.param( + "Qwen/Qwen3.5-0.8B-Base", + { + "method": "mtp", + "num_speculative_tokens": 3, + }, + id="mtp", + ), +] + +# Measured 1.85 (eagle3), 1.99 (dflash), 2.29 (mtp) on CPU; this floor leaves +# room for numeric drift while still failing if speculation stops paying off. +MIN_ACCEPTANCE_LEN = 1.5 + + +@pytest.mark.skipif( + not HAS_TRITON, reason="the V2 runner needs triton-cpu, which CI builds" +) +@pytest.mark.parametrize("model,speculative_config", SMOKE_CONFIGS) +def test_v2_speculator_smoke(model: str, speculative_config: dict): + """Gates that drafts are accepted often enough for speculation to pay off.""" + llm = _make_llm(model=model, speculative_config=speculative_config) + try: + assert llm.llm_engine.vllm_config.use_v2_model_runner, ( + f"{speculative_config['method']} fell back to the V1 runner; " + "this test is meant to cover the V2 path" + ) + llm.generate(SMOKE_PROMPTS, SMOKE_SP) + acceptance = compute_acceptance_len(llm.get_metrics()) + assert acceptance >= MIN_ACCEPTANCE_LEN, ( + f"acceptance length {acceptance:.3f} below {MIN_ACCEPTANCE_LEN}; " + "drafts are being rejected far more than expected" + ) + finally: + del llm