[Bugfix][Spec Decode] Cap DFlash/DSpark profiling query batch (#56448)

Signed-off-by: wangyicong <wangyicong@bytedance.com>
This commit is contained in:
Yicong Wang
2026-09-22 12:36:32 +08:00
committed by GitHub
parent ff3c9cba3f
commit c4d424c2e2
3 changed files with 105 additions and 0 deletions
+5
View File
@@ -790,6 +790,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_tokens = max(num_tokens, self.decode_query_len)
num_reqs = num_tokens // self.decode_query_len
assert num_tokens % self.decode_query_len == 0
elif self.speculator is not None:
num_reqs = min(
num_reqs,
self.max_num_tokens // self.speculator.num_query_per_req,
)
# Distribute the remainder evenly so no dummy request exceeds
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
num_tokens_per_request = [