[SpecDecode] Add LiLiCorr drafter (#57934)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Codex
Benjamin Chislett
parent
25b0add7b8
commit
73a78e6f1f
@@ -108,26 +108,93 @@ def _cache_draft_logits_kernel(
|
||||
tl.store(cached_candidate_ptr + cache_base + offsets, token_ids, mask=mask)
|
||||
|
||||
|
||||
class CandidateSampler:
|
||||
"""The shared DFlash2/LiLiCorr candidate walk and realized proposal cache."""
|
||||
|
||||
def __init__(
|
||||
self, max_num_reqs: int, num_steps: int, top_k: int, device: torch.device
|
||||
):
|
||||
self.num_steps = num_steps
|
||||
self.top_k = top_k
|
||||
self.scores = torch.empty(
|
||||
max_num_reqs, num_steps, top_k, dtype=torch.float32, device=device
|
||||
)
|
||||
self.cached_candidate_ids = torch.zeros(
|
||||
self.scores.shape, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
def sample(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
scores: torch.Tensor,
|
||||
num_reqs: int,
|
||||
sample_pos: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
use_fp64: bool,
|
||||
) -> None:
|
||||
candidate_ids = candidate_ids.contiguous()
|
||||
sample_pos = sample_pos.contiguous()
|
||||
idx_mapping = idx_mapping.contiguous()
|
||||
block_k = triton.next_power_of_2(self.top_k)
|
||||
_selector_walk_kernel[(num_reqs,)](
|
||||
scores.contiguous(),
|
||||
candidate_ids.contiguous(),
|
||||
sample_pos,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
draft_tokens,
|
||||
self.scores,
|
||||
num_steps=self.num_steps,
|
||||
top_k=self.top_k,
|
||||
BLOCK_K=block_k,
|
||||
SAMPLE_PROBABILISTIC=draft_logits is not None,
|
||||
USE_FP64=use_fp64,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
if draft_logits is not None:
|
||||
self._cache_draft_logits(
|
||||
candidate_ids, num_reqs * self.num_steps, idx_mapping, draft_logits
|
||||
)
|
||||
|
||||
def _cache_draft_logits(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
num_sample: int,
|
||||
idx_mapping: torch.Tensor,
|
||||
draft_logits: torch.Tensor,
|
||||
) -> None:
|
||||
block_k = triton.next_power_of_2(self.top_k)
|
||||
_cache_draft_logits_kernel[(num_sample,)](
|
||||
draft_logits,
|
||||
self.cached_candidate_ids,
|
||||
candidate_ids,
|
||||
self.scores,
|
||||
idx_mapping,
|
||||
draft_logits.stride(0),
|
||||
draft_logits.stride(1),
|
||||
num_steps=self.num_steps,
|
||||
top_k=self.top_k,
|
||||
BLOCK_K=block_k,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
class DFlash2Speculator(DFlashSpeculator):
|
||||
_speculator_name = "DFlash2"
|
||||
_candidate_top_k_key = "selector_top_k"
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
draft_config = self.draft_model_config.hf_config.dflash_config
|
||||
self.selector_top_k = int(draft_config["selector_top_k"])
|
||||
self._anchor_indices = (
|
||||
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
* self.num_query_per_req
|
||||
)
|
||||
self._selector_scores = torch.empty(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.selector_top_k,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self._cached_candidate_ids = torch.zeros(
|
||||
self._selector_scores.shape, dtype=torch.int64, device=device
|
||||
self.top_k = int(draft_config[self._candidate_top_k_key])
|
||||
self.candidate_sampler = CandidateSampler(
|
||||
self.max_num_reqs, self.num_speculative_steps, self.top_k, device
|
||||
)
|
||||
|
||||
def draft_logits_spec(self, vllm_config: VllmConfig) -> tuple[torch.dtype, float]:
|
||||
@@ -136,48 +203,6 @@ class DFlash2Speculator(DFlashSpeculator):
|
||||
# candidates.
|
||||
return torch.float32, -float("inf")
|
||||
|
||||
def _sample_path(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
scores: torch.Tensor,
|
||||
num_reqs: int,
|
||||
) -> None:
|
||||
block_k = triton.next_power_of_2(self.selector_top_k)
|
||||
_selector_walk_kernel[(num_reqs,)](
|
||||
scores.contiguous(),
|
||||
candidate_ids.contiguous(),
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.draft_tokens,
|
||||
self._selector_scores,
|
||||
num_steps=self.num_speculative_steps,
|
||||
top_k=self.selector_top_k,
|
||||
BLOCK_K=block_k,
|
||||
SAMPLE_PROBABILISTIC=self.draft_logits is not None,
|
||||
USE_FP64=self.use_fp64_gumbel,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
def _cache_draft_logits(self, candidate_ids: torch.Tensor, num_sample: int) -> None:
|
||||
draft_logits = self.draft_logits
|
||||
assert draft_logits is not None
|
||||
block_k = triton.next_power_of_2(self.selector_top_k)
|
||||
_cache_draft_logits_kernel[(num_sample,)](
|
||||
draft_logits,
|
||||
self._cached_candidate_ids,
|
||||
candidate_ids,
|
||||
self._selector_scores,
|
||||
self.sample_idx_mapping,
|
||||
draft_logits.stride(0),
|
||||
draft_logits.stride(1),
|
||||
num_steps=self.num_speculative_steps,
|
||||
top_k=self.selector_top_k,
|
||||
BLOCK_K=block_k,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
@@ -202,22 +227,39 @@ class DFlash2Speculator(DFlashSpeculator):
|
||||
hidden_states.flatten(0, 1)
|
||||
)
|
||||
candidate_ids = candidate_ids.view(
|
||||
num_reqs, self.num_speculative_steps, self.selector_top_k
|
||||
num_reqs, self.num_speculative_steps, self.top_k
|
||||
)
|
||||
unary_logits = unary_logits.view_as(candidate_ids)
|
||||
anchor_token_ids = self.input_buffers.input_ids[self._anchor_indices[:num_reqs]]
|
||||
scores = self.model.model.candidate_selector(
|
||||
scores = self._score_candidates(candidate_ids, unary_logits, hidden_states)
|
||||
self.candidate_sampler.sample(
|
||||
candidate_ids,
|
||||
unary_logits,
|
||||
hidden_states,
|
||||
anchor_token_ids,
|
||||
scores,
|
||||
num_reqs,
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.draft_tokens,
|
||||
self.draft_logits,
|
||||
self.use_fp64_gumbel,
|
||||
)
|
||||
self._sample_path(candidate_ids, scores, num_reqs)
|
||||
if self.enable_adaptive_verification:
|
||||
self._maybe_predict_acceptance(
|
||||
self._selector_scores[:num_reqs].flatten(0, 1),
|
||||
self.candidate_sampler.scores[:num_reqs].flatten(0, 1),
|
||||
self.sample_idx_mapping[:num_sample],
|
||||
self.sample_col[:num_sample],
|
||||
)
|
||||
if self.draft_logits is not None:
|
||||
self._cache_draft_logits(candidate_ids, num_sample)
|
||||
|
||||
def _score_candidates(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
unary_logits: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = candidate_ids.shape[0]
|
||||
anchor_token_ids = self.input_buffers.input_ids[
|
||||
: num_reqs * self.num_query_per_req : self.num_query_per_req
|
||||
]
|
||||
return self.model.model.candidate_selector(
|
||||
candidate_ids, unary_logits, hidden_states, anchor_token_ids
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user