[Doc] Explicitly state that PP isn't compatible with speculative decoding yet (#10975)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2024-12-07 17:20:49 +00:00
committed by GitHub
parent 39e227c7ae
commit c889d5888b
8 changed files with 32 additions and 9 deletions
+4
View File
@@ -54,6 +54,10 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
speculative_config: SpeculativeConfig = vllm_config.speculative_config
assert speculative_config is not None
if vllm_config.parallel_config.pipeline_parallel_size > 1:
raise NotImplementedError("Speculative decoding is currently "
"incompatible with pipeline parallelism")
draft_worker_kwargs = kwargs.copy()
kwargs["model_runner_cls"] = TargetModelRunner