[CI] Stabilize speculator memory teardown (#50284)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
Andreas Karatzas
2026-07-30 18:42:55 +08:00
committed by GitHub
parent 165ed33327
commit 61c1d098e5
+37 -29
View File
@@ -379,6 +379,7 @@ def test_speculators_model_integration(
sampling_config: SamplingParams,
model_path: str,
expected_accuracy_threshold: float,
vllm_runner,
):
"""
Test that speculators models work with the simplified integration.
@@ -401,41 +402,48 @@ def test_speculators_model_integration(
test_prompts = get_test_prompts(mm_enabled=False)
# First run: Direct speculator model (simplified integration)
spec_llm = LLM(model=model_path, max_model_len=4096, gpu_memory_utilization=0.92)
evaluate_llm_for_gsm8k(
spec_llm, expected_accuracy_threshold=expected_accuracy_threshold
)
spec_outputs = spec_llm.chat(test_prompts, sampling_config)
with vllm_runner(
model_path,
trust_remote_code=False,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
max_model_len=4096,
gpu_memory_utilization=0.92,
) as spec_runner:
evaluate_llm_for_gsm8k(
spec_runner.llm, expected_accuracy_threshold=expected_accuracy_threshold
)
spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
# Verify speculative config was auto-detected
assert spec_llm.llm_engine.vllm_config.speculative_config is not None, (
f"Speculative config should be auto-detected for {model_path}"
)
# Verify speculative config was auto-detected
assert spec_runner.llm.llm_engine.vllm_config.speculative_config is not None, (
f"Speculative config should be auto-detected for {model_path}"
)
spec_config = spec_llm.llm_engine.vllm_config.speculative_config
assert spec_config.num_speculative_tokens > 0, (
f"Expected positive speculative tokens, "
f"got {spec_config.num_speculative_tokens}"
)
spec_config = spec_runner.llm.llm_engine.vllm_config.speculative_config
assert spec_config.num_speculative_tokens > 0, (
f"Expected positive speculative tokens, "
f"got {spec_config.num_speculative_tokens}"
)
# Verify draft model is set to the speculator model
assert spec_config.model == model_path, (
f"Draft model should be {model_path}, got {spec_config.model}"
)
# Verify draft model is set to the speculator model
assert spec_config.model == model_path, (
f"Draft model should be {model_path}, got {spec_config.model}"
)
# Extract verifier model for reference run
verifier_model = spec_llm.llm_engine.vllm_config.model_config.model
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
# Extract verifier model for reference run
verifier_model = spec_runner.llm.llm_engine.vllm_config.model_config.model
# Second run: Reference without speculative decoding
ref_llm = LLM(model=verifier_model, max_model_len=4096, gpu_memory_utilization=0.92)
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
del ref_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
with vllm_runner(
verifier_model,
trust_remote_code=False,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
max_model_len=4096,
gpu_memory_utilization=0.92,
) as ref_runner:
ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
# Compare outputs
matches = sum(