[BugFix] Reserve the bonus query slot in DFlash scheduling budget (#51256)
Signed-off-by: HF-001 <1670186653@qq.com>
This commit is contained in:
@@ -490,6 +490,30 @@ def test_draft_model_enables_async_scheduling_by_default():
|
|||||||
assert cfg.scheduler_config.async_scheduling is True
|
assert cfg.scheduler_config.async_scheduling is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "parallel_drafting", "expected_slots"),
|
||||||
|
[
|
||||||
|
pytest.param("eagle3", False, 0, id="eagle3"),
|
||||||
|
pytest.param("eagle3", True, 7, id="p-eagle"),
|
||||||
|
pytest.param("dflash", True, 8, id="dflash"),
|
||||||
|
pytest.param("dspark", True, 7, id="dspark"),
|
||||||
|
pytest.param("mtp", False, 0, id="mtp"),
|
||||||
|
pytest.param("ngram", False, 0, id="ngram"),
|
||||||
|
pytest.param("draft_model", False, 1, id="draft-model"),
|
||||||
|
pytest.param("draft_model", True, 8, id="pard"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_max_num_new_slots_for_drafting(method, parallel_drafting, expected_slots):
|
||||||
|
speculative_config = SpeculativeConfig(
|
||||||
|
model="ngram",
|
||||||
|
num_speculative_tokens=8,
|
||||||
|
)
|
||||||
|
speculative_config.method = method
|
||||||
|
speculative_config.parallel_drafting = parallel_drafting
|
||||||
|
|
||||||
|
assert speculative_config.max_num_new_slots_for_drafting == expected_slots
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class _TestConfigFields:
|
class _TestConfigFields:
|
||||||
a: int
|
a: int
|
||||||
|
|||||||
+35
-10
@@ -1448,19 +1448,44 @@ class SpeculativeConfig:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def max_num_new_slots_for_drafting(self) -> int:
|
def max_num_new_slots_for_drafting(self) -> int:
|
||||||
|
"""Return the maximum additional drafting slots per request.
|
||||||
|
|
||||||
|
The scheduler budget already includes one query slot per decoding request.
|
||||||
|
Let K be ``num_speculative_tokens``. Standard configurations require:
|
||||||
|
|
||||||
|
==================== ============= ======== ================
|
||||||
|
Algorithm Method Parallel Additional slots
|
||||||
|
==================== ============= ======== ================
|
||||||
|
EAGLE3 eagle3 No 0
|
||||||
|
P-EAGLE eagle3 Yes K - 1
|
||||||
|
DFlash dflash Yes K
|
||||||
|
DSpark dspark Yes K - 1
|
||||||
|
MTP mtp No 0
|
||||||
|
N-gram ngram No 0
|
||||||
|
Draft model draft_model No 1
|
||||||
|
PARD draft_model Yes K
|
||||||
|
==================== ============= ======== ================
|
||||||
"""
|
"""
|
||||||
Calculate the maximum number of new slots that might be added to the batch
|
num_draft_tokens = self.num_speculative_tokens
|
||||||
when drafting.
|
|
||||||
"""
|
if self.use_dflash():
|
||||||
slots_per_req = 0 # for serial non-draft-model methods, no change needed
|
# DFlash uses one bonus query followed by K mask queries.
|
||||||
|
return num_draft_tokens
|
||||||
|
|
||||||
if self.parallel_drafting:
|
if self.parallel_drafting:
|
||||||
# For parallel drafting, we need one new slot per 'masked' token
|
if self.uses_draft_model():
|
||||||
slots_per_req = self.num_speculative_tokens - 1
|
# PARD does not shift the existing input, so all K query
|
||||||
|
# positions require additional slots.
|
||||||
|
return num_draft_tokens
|
||||||
|
|
||||||
|
# The existing query is reused; only masked queries need new slots.
|
||||||
|
return num_draft_tokens - 1
|
||||||
|
|
||||||
if self.uses_draft_model():
|
if self.uses_draft_model():
|
||||||
# For draft model-based speculation, we need one new slot per request
|
# The autoregressive draft-model input retains one unsliced token.
|
||||||
# Since we do not slice the draft tokens
|
return 1
|
||||||
slots_per_req += 1
|
|
||||||
return slots_per_req
|
return 0
|
||||||
|
|
||||||
def use_gemma4_mtp(self) -> bool:
|
def use_gemma4_mtp(self) -> bool:
|
||||||
return (
|
return (
|
||||||
|
|||||||
Reference in New Issue
Block a user