[PCP][Spec Decode] Adds PCP support for single-module MTP and replicated DSpark. (#56107)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Summer Yang
OpenAI Codex
OpenAI Codex
parent
e7edf17cea
commit
980c16c8e4
@@ -656,7 +656,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
self.supports_mm_inputs,
|
||||
self.req_states,
|
||||
self.block_tables,
|
||||
cls=self.pcp_manager_cls,
|
||||
)
|
||||
@@ -668,6 +667,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kv_cache_config,
|
||||
self.max_num_reqs,
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.pcp_manager = self.pcp_manager
|
||||
initialize_mamba_ssu_backend(
|
||||
self.vllm_config.mamba_config,
|
||||
self.kv_cache_config,
|
||||
@@ -697,7 +698,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.fast_prefill = FastPrefillHelper(
|
||||
self.cudagraph_manager, self.max_num_tokens
|
||||
)
|
||||
check_attention_cp_compatibility(self.vllm_config)
|
||||
check_attention_cp_compatibility(self.vllm_config, target_attn_layer_names)
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
# HACK(woosuk)
|
||||
self.speculator.set_attn(
|
||||
@@ -1964,9 +1965,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
|
||||
|
||||
# Last rank: sample tokens
|
||||
draft_hidden_states = hidden_states
|
||||
assert draft_hidden_states is not None
|
||||
hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling(
|
||||
self.pcp_manager, hidden_states, input_batch
|
||||
)
|
||||
if self.pcp_manager is not None and aux_hidden_states is not None:
|
||||
aux_hidden_states = [
|
||||
self.pcp_manager.restore_hidden_states(states)
|
||||
for states in aux_hidden_states
|
||||
]
|
||||
|
||||
sampler_output, num_sampled, num_rejected = self.sample(
|
||||
hidden_states, input_batch, grammar_output
|
||||
@@ -2043,10 +2051,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
|
||||
# target returns a persistent buffer sized at max_num_batched_tokens;
|
||||
# slice to the active token count that propose() expects.
|
||||
spec_hidden_states = hidden_states
|
||||
spec_hidden_states = draft_hidden_states
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
draft_tokens = self.speculator.propose(
|
||||
input_batch,
|
||||
|
||||
Reference in New Issue
Block a user