[Dynamic Spec Decoding] Auto-disable by the running queue size (#4592)

Co-authored-by: Cade Daniel <edacih@gmail.com>
This commit is contained in:
Cody Yu
2024-05-08 21:44:00 +00:00
committed by GitHub
co-authored by Cade Daniel
parent 89579a201f
commit f942efb5a3
11 changed files with 227 additions and 39 deletions
+2
View File
@@ -93,6 +93,8 @@ class GPUExecutor(ExecutorBase):
spec_decode_worker = SpecDecodeWorker.create_worker(
scorer_worker=target_worker,
draft_worker_kwargs=draft_worker_kwargs,
disable_by_batch_size=self.speculative_config.
speculative_disable_by_batch_size,
)
assert self.parallel_config.world_size == 1, (