[Model] Add DiffusionGemma Support (#45163)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Martin Kukla <martin.kukla@cantab.net> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Dipika Sikka <dsikka@redhat.com> Co-authored-by: NickLucche <nlucches@redhat.com> Co-authored-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: Alec Kohlhoff <134344302+aleckohlhoff@users.noreply.github.com> Co-authored-by: Porras Huang <20535584+porrashuang@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: scoootscooob <167050519+scoootscooob@users.noreply.github.com>
This commit is contained in:
co-authored by
Martin Kukla
Matthew Bonanni
Dipika Sikka
NickLucche
jiahanc
Alec Kohlhoff
Porras Huang
Claude
scoootscooob
parent
1ce3cdc5c1
commit
eb28452b10
@@ -302,6 +302,7 @@ def _combine_sampled_and_draft_tokens_kernel(
|
||||
cu_num_logits_ptr,
|
||||
logits_indices_ptr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
NUM_NEW_SAMPLED_TOKENS: tl.constexpr = 1,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
@@ -310,7 +311,7 @@ def _combine_sampled_and_draft_tokens_kernel(
|
||||
cu_num_logits_start = tl.load(cu_num_logits_ptr + batch_idx)
|
||||
cu_num_logits_end = tl.load(cu_num_logits_ptr + batch_idx + 1)
|
||||
num_logits = cu_num_logits_end - cu_num_logits_start
|
||||
num_draft_tokens = num_logits - 1
|
||||
num_draft_tokens = num_logits - NUM_NEW_SAMPLED_TOKENS
|
||||
|
||||
# Compute the logits indices.
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
@@ -328,9 +329,10 @@ def _combine_sampled_and_draft_tokens_kernel(
|
||||
# Handling prefill tokens. No sampled or draft tokens.
|
||||
return
|
||||
|
||||
# Write the last sampled token ID to input_ids.
|
||||
last_token_id = tl.load(last_sampled_tokens_ptr + req_state_idx)
|
||||
tl.store(input_ids_ptr + query_end - num_logits, last_token_id)
|
||||
if NUM_NEW_SAMPLED_TOKENS > 0:
|
||||
# Write the last sampled token ID to input_ids.
|
||||
last_token_id = tl.load(last_sampled_tokens_ptr + req_state_idx)
|
||||
tl.store(input_ids_ptr + query_end - num_logits, last_token_id)
|
||||
|
||||
# Write the draft tokens (if any) to input_ids.
|
||||
if num_draft_tokens > 0:
|
||||
@@ -356,7 +358,11 @@ def combine_sampled_and_draft_tokens(
|
||||
draft_tokens: torch.Tensor,
|
||||
cu_num_logits: torch.Tensor,
|
||||
num_logits: int,
|
||||
num_new_sampled_tokens: int = 1, # excl accepted draft tokens, a.k.a bonus tokens
|
||||
) -> torch.Tensor:
|
||||
assert num_new_sampled_tokens in (0, 1), (
|
||||
f"num_new_sampled_tokens must be 0 or 1, got {num_new_sampled_tokens}"
|
||||
)
|
||||
# use idx_mapping.shape[0] for actual request count
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
num_speculative_steps = draft_tokens.shape[-1]
|
||||
@@ -377,9 +383,12 @@ def combine_sampled_and_draft_tokens(
|
||||
draft_tokens.stride(0),
|
||||
cu_num_logits,
|
||||
logits_indices,
|
||||
# NOTE(woosuk): Add 1 to ensure the block can cover the last sampled token
|
||||
# in addition to all draft tokens.
|
||||
BLOCK_SIZE=triton.next_power_of_2(num_speculative_steps + 1),
|
||||
NUM_NEW_SAMPLED_TOKENS=num_new_sampled_tokens,
|
||||
# NOTE(woosuk): Add num_new_sampled_tokens to ensure the block covers the
|
||||
# last sampled token in addition to all draft tokens.
|
||||
BLOCK_SIZE=triton.next_power_of_2(
|
||||
num_speculative_steps + num_new_sampled_tokens
|
||||
),
|
||||
)
|
||||
return logits_indices
|
||||
|
||||
|
||||
Reference in New Issue
Block a user