[watermarking] add context deduplication support to speculative decoding (#56807)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Simon Veitner
Lucas Wilkinson
parent
73a5831127
commit
0f8b398158
@@ -220,6 +220,8 @@ def gumbel_block_argmax(
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
vocab_size,
|
||||
IS_DRAFTING: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
@@ -240,12 +242,16 @@ def gumbel_block_argmax(
|
||||
col = tl.load(logits_cache_col_ptr + token_idx)
|
||||
else:
|
||||
col = tl.load(logits_cache_col_ptr)
|
||||
cached_logits = tl.load(
|
||||
logits_cache_source_ptr + token_idx * logits_cache_source_stride + block,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
logits_cache_ptr
|
||||
+ req_state_idx * logits_cache_stride_0
|
||||
+ col * logits_cache_stride_1
|
||||
+ block,
|
||||
logits,
|
||||
cached_logits,
|
||||
mask=mask & is_valid_req,
|
||||
)
|
||||
|
||||
@@ -275,6 +281,8 @@ def _gumbel_sample_kernel(
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
expanded_idx_mapping_ptr,
|
||||
@@ -312,6 +320,8 @@ def _gumbel_sample_kernel(
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
vocab_size,
|
||||
IS_DRAFTING=IS_DRAFTING,
|
||||
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
|
||||
@@ -334,6 +344,7 @@ def gumbel_sample(
|
||||
logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size]
|
||||
logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens]
|
||||
use_fp64: bool = False,
|
||||
logits_cache_source: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Enforce contiguity on non-strided input tensors
|
||||
expanded_idx_mapping = expanded_idx_mapping.contiguous()
|
||||
@@ -342,11 +353,26 @@ def gumbel_sample(
|
||||
logits_cache_col = logits_cache_col.contiguous()
|
||||
num_tokens, vocab_size = logits.shape
|
||||
if logits_cache is not None:
|
||||
if logits_cache_source is None:
|
||||
logits_cache_source = logits
|
||||
assert logits_cache_source.shape == logits.shape, (
|
||||
"logits cache source must match sampled logits shape"
|
||||
)
|
||||
assert logits_cache_source.device == logits.device, (
|
||||
"logits cache source must be on the sampled logits device"
|
||||
)
|
||||
assert logits_cache_source.dtype == logits_cache.dtype, (
|
||||
"logits cache source and destination must have the same dtype"
|
||||
)
|
||||
assert logits_cache.size(-1) >= vocab_size, (
|
||||
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
|
||||
f"than the sampled logits ({vocab_size}). Cached logits would be "
|
||||
"truncated."
|
||||
)
|
||||
elif logits_cache_source is not None:
|
||||
raise ValueError("logits_cache_source requires logits_cache")
|
||||
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
|
||||
logits_cache_source = logits_cache_source.contiguous()
|
||||
BLOCK_SIZE = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
|
||||
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
|
||||
@@ -362,6 +388,8 @@ def gumbel_sample(
|
||||
logits_cache.stride(0) if logits_cache is not None else 0,
|
||||
logits_cache.stride(1) if logits_cache is not None else 0,
|
||||
logits_cache_col,
|
||||
logits_cache_source,
|
||||
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
|
||||
logits,
|
||||
logits.stride(0),
|
||||
expanded_idx_mapping,
|
||||
|
||||
@@ -51,12 +51,25 @@ def _repeated_context_mask_kernel(
|
||||
req_indices_ptr,
|
||||
prompt_lens_ptr,
|
||||
total_lens_ptr,
|
||||
history_offsets_ptr,
|
||||
history_offsets_stride,
|
||||
contexts_ptr,
|
||||
context_stride,
|
||||
local_positions_ptr,
|
||||
prior_contexts_ptr,
|
||||
prior_contexts_stride_0,
|
||||
prior_contexts_stride_1,
|
||||
steps_ptr,
|
||||
steps_stride,
|
||||
enabled_ptr,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
NUM_SPECULATIVE_STEPS: tl.constexpr,
|
||||
MAX_HISTORY: tl.constexpr,
|
||||
INCLUDE_PROMPT: tl.constexpr,
|
||||
SKIP_PARTIAL_CONTEXT: tl.constexpr,
|
||||
HAS_HISTORY_OFFSETS: tl.constexpr,
|
||||
SPECULATIVE_CONTEXTS: tl.constexpr,
|
||||
DRAFT_CONTEXTS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
@@ -71,12 +84,16 @@ def _repeated_context_mask_kernel(
|
||||
return
|
||||
sequence_start = 0 if INCLUDE_PROMPT else prompt_len
|
||||
history_len = total_len - sequence_start
|
||||
history_offset = 0
|
||||
if HAS_HISTORY_OFFSETS:
|
||||
history_offset = tl.load(history_offsets_ptr + row * history_offsets_stride)
|
||||
|
||||
offsets = tl.arange(0, BLOCK)
|
||||
repeated = tl.full((), 0, tl.int32)
|
||||
scan_start = 0
|
||||
if MAX_HISTORY > 0:
|
||||
scan_start = tl.maximum(history_len - MAX_HISTORY, 0)
|
||||
remaining_history = tl.maximum(MAX_HISTORY - history_offset, 0)
|
||||
scan_start = tl.maximum(history_len - remaining_history, 0)
|
||||
# Aligned loop bounds let the per-lane offsets fold into immediates.
|
||||
aligned_start = (scan_start // BLOCK) * BLOCK
|
||||
for block_start in tl.range(aligned_start, history_len, BLOCK):
|
||||
@@ -96,6 +113,93 @@ def _repeated_context_mask_kernel(
|
||||
)
|
||||
matches &= historical_token == context_token
|
||||
repeated |= tl.max(matches.to(tl.int32), axis=0)
|
||||
|
||||
if SPECULATIVE_CONTEXTS:
|
||||
local_pos = tl.load(local_positions_ptr + row)
|
||||
if INCLUDE_PROMPT:
|
||||
partial_context = tl.full((), 0, tl.int32)
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
partial_context |= context_token < 0
|
||||
repeated |= partial_context
|
||||
for offset in range(1, NUM_SPECULATIVE_STEPS + 1):
|
||||
prior_row = row - offset
|
||||
in_history = (prior_row >= 0) & (local_pos >= offset)
|
||||
if MAX_HISTORY > 0:
|
||||
in_history &= offset <= MAX_HISTORY
|
||||
prior_req_idx = tl.load(
|
||||
req_indices_ptr + tl.maximum(prior_row, 0),
|
||||
mask=in_history,
|
||||
other=-1,
|
||||
)
|
||||
prior_local_pos = tl.load(
|
||||
local_positions_ptr + tl.maximum(prior_row, 0),
|
||||
mask=in_history,
|
||||
other=-1,
|
||||
)
|
||||
matches = (
|
||||
in_history
|
||||
& (req_idx == prior_req_idx)
|
||||
& (local_pos == prior_local_pos + offset)
|
||||
)
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
prior_token = tl.load(
|
||||
contexts_ptr
|
||||
+ tl.maximum(prior_row, 0) * context_stride
|
||||
+ context_offset,
|
||||
mask=in_history,
|
||||
other=-2,
|
||||
)
|
||||
matches &= context_token == prior_token
|
||||
repeated |= matches
|
||||
|
||||
if DRAFT_CONTEXTS:
|
||||
step = tl.load(steps_ptr + row * steps_stride).to(tl.int64)
|
||||
partial_context = tl.full((), 0, tl.int32)
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
partial_context |= context_token < 0
|
||||
if INCLUDE_PROMPT:
|
||||
repeated |= partial_context
|
||||
|
||||
for prior_step in range(NUM_SPECULATIVE_STEPS):
|
||||
in_history = prior_step < step
|
||||
if MAX_HISTORY > 0:
|
||||
in_history &= prior_step >= step - MAX_HISTORY
|
||||
matches = in_history
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
prior_token = tl.load(
|
||||
prior_contexts_ptr
|
||||
+ row * prior_contexts_stride_0
|
||||
+ prior_step * prior_contexts_stride_1
|
||||
+ context_offset
|
||||
)
|
||||
matches &= context_token == prior_token
|
||||
repeated |= matches
|
||||
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
tl.store(
|
||||
prior_contexts_ptr
|
||||
+ row * prior_contexts_stride_0
|
||||
+ step * prior_contexts_stride_1
|
||||
+ context_offset,
|
||||
context_token,
|
||||
)
|
||||
enabled = tl.load(enabled_ptr + row)
|
||||
repeated = enabled & (repeated == 0)
|
||||
tl.store(output_ptr + row, repeated)
|
||||
|
||||
|
||||
@@ -108,6 +212,7 @@ def _repeated_context_mask_cpu(
|
||||
max_history: int | None = None,
|
||||
include_prompt: bool = False,
|
||||
skip_partial_context: bool = False,
|
||||
history_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Reference implementation; the parity tests check the Triton kernel against it."""
|
||||
repeated = torch.zeros(len(req_indices), dtype=torch.bool)
|
||||
@@ -124,8 +229,11 @@ def _repeated_context_mask_cpu(
|
||||
history_tokens = all_token_ids[req_idx, sequence_start:total_len].tolist()
|
||||
prefix = [-1] * contexts.shape[-1]
|
||||
current_context = tuple(contexts[row].tolist())
|
||||
history_offset = 0 if history_offsets is None else int(history_offsets[row])
|
||||
history_start = (
|
||||
max(0, len(history_tokens) - max_history) if max_history is not None else 0
|
||||
max(0, len(history_tokens) - max(0, max_history - history_offset))
|
||||
if max_history is not None
|
||||
else 0
|
||||
)
|
||||
for history_pos, token_id in enumerate(history_tokens):
|
||||
if history_pos >= history_start and (
|
||||
@@ -146,6 +254,9 @@ def repeated_context_mask(
|
||||
max_history: int | None = None,
|
||||
include_prompt: bool = False,
|
||||
skip_partial_context: bool = False,
|
||||
history_offsets: torch.Tensor | None = None,
|
||||
local_positions: torch.Tensor | None = None,
|
||||
num_speculative_steps: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Return, per row, whether the row's context already occurred in its history.
|
||||
|
||||
@@ -163,12 +274,23 @@ def repeated_context_mask(
|
||||
include_prompt: Search the prompt as well as the generated tokens.
|
||||
skip_partial_context: Mark contexts containing start padding so they use
|
||||
ordinary sampling.
|
||||
history_offsets: Number of newer, non-committed positions preceding each
|
||||
context. These positions count toward `max_history`. Must be 1-D
|
||||
over rows; a stride-0 broadcast row is fine.
|
||||
local_positions: Position within each request's speculative block. When
|
||||
provided, compare each context with earlier contexts in that block.
|
||||
num_speculative_steps: Maximum number of draft tokens in the block.
|
||||
|
||||
"""
|
||||
if max_history is not None and max_history < 1:
|
||||
raise ValueError("max_history must be positive or None")
|
||||
if (local_positions is None) != (num_speculative_steps == 0):
|
||||
raise ValueError(
|
||||
"local_positions and positive num_speculative_steps must be "
|
||||
"provided together"
|
||||
)
|
||||
if all_token_ids.device.type == "cpu":
|
||||
return _repeated_context_mask_cpu(
|
||||
repeated = _repeated_context_mask_cpu(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
@@ -177,10 +299,42 @@ def repeated_context_mask(
|
||||
max_history,
|
||||
include_prompt,
|
||||
skip_partial_context,
|
||||
history_offsets,
|
||||
)
|
||||
if local_positions is not None:
|
||||
max_offset = num_speculative_steps + 1
|
||||
if max_history is not None:
|
||||
max_offset = min(max_offset, max_history + 1)
|
||||
for offset in range(1, max_offset):
|
||||
prior_contexts = torch.cat(
|
||||
(contexts[:offset], contexts[:-offset]), dim=0
|
||||
)
|
||||
prior_requests = torch.cat(
|
||||
(req_indices[:offset], req_indices[:-offset]), dim=0
|
||||
)
|
||||
prior_local_pos = torch.cat(
|
||||
(local_positions[:offset], local_positions[:-offset]), dim=0
|
||||
)
|
||||
repeated |= (
|
||||
(req_indices >= 0)
|
||||
& (local_positions >= offset)
|
||||
& (req_indices == prior_requests)
|
||||
& (local_positions == prior_local_pos + offset)
|
||||
& (contexts == prior_contexts).all(dim=-1)
|
||||
)
|
||||
if include_prompt:
|
||||
repeated |= (contexts < 0).any(dim=-1)
|
||||
return repeated
|
||||
|
||||
if contexts.stride(-1) != 1:
|
||||
contexts = contexts.contiguous()
|
||||
# Only history_offsets passes its row stride to the kernel.
|
||||
req_indices = req_indices.contiguous()
|
||||
if local_positions is not None:
|
||||
local_positions = local_positions.contiguous()
|
||||
# `prompt_lens` and `total_lens` are indexed flat by request slot, and
|
||||
# `all_token_ids` takes a row stride but is read flat within the row. Those
|
||||
# three are contiguous by contract; every caller owns them outright.
|
||||
repeated = torch.empty(len(req_indices), dtype=torch.bool, device=contexts.device)
|
||||
_repeated_context_mask_kernel[(len(req_indices),)](
|
||||
repeated,
|
||||
@@ -189,17 +343,104 @@ def repeated_context_mask(
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
history_offsets,
|
||||
0 if history_offsets is None else history_offsets.stride(0),
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
local_positions,
|
||||
None,
|
||||
0,
|
||||
0,
|
||||
None,
|
||||
0,
|
||||
None,
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
NUM_SPECULATIVE_STEPS=num_speculative_steps,
|
||||
MAX_HISTORY=0 if max_history is None else max_history,
|
||||
INCLUDE_PROMPT=include_prompt,
|
||||
SKIP_PARTIAL_CONTEXT=skip_partial_context,
|
||||
HAS_HISTORY_OFFSETS=history_offsets is not None,
|
||||
SPECULATIVE_CONTEXTS=local_positions is not None,
|
||||
DRAFT_CONTEXTS=False,
|
||||
BLOCK=512,
|
||||
)
|
||||
return repeated
|
||||
|
||||
|
||||
def draft_watermarking_mask(
|
||||
all_token_ids: torch.Tensor,
|
||||
req_indices: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
total_lens: torch.Tensor,
|
||||
prior_contexts: torch.Tensor,
|
||||
contexts: torch.Tensor,
|
||||
steps: torch.Tensor,
|
||||
enabled: torch.Tensor,
|
||||
max_history: int | None,
|
||||
include_prompt: bool,
|
||||
) -> torch.Tensor:
|
||||
if contexts.device.type == "cpu":
|
||||
repeated = repeated_context_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
contexts,
|
||||
max_history,
|
||||
include_prompt=include_prompt,
|
||||
history_offsets=steps,
|
||||
)
|
||||
prior_steps = torch.arange(prior_contexts.shape[1]).unsqueeze(0)
|
||||
in_history = prior_steps < steps.unsqueeze(1)
|
||||
if max_history is not None:
|
||||
in_history &= prior_steps >= steps.unsqueeze(1) - max_history
|
||||
repeated |= (
|
||||
(prior_contexts[: len(contexts)] == contexts.unsqueeze(1)).all(dim=-1)
|
||||
& in_history
|
||||
).any(dim=-1)
|
||||
if include_prompt:
|
||||
repeated |= (contexts < 0).any(dim=-1)
|
||||
row_indices = torch.arange(len(contexts))
|
||||
prior_contexts.index_put_((row_indices, steps), contexts)
|
||||
return enabled & ~repeated
|
||||
|
||||
if contexts.stride(-1) != 1:
|
||||
contexts = contexts.contiguous()
|
||||
# Only `steps` passes its row stride to the kernel.
|
||||
req_indices = req_indices.contiguous()
|
||||
enabled = enabled.contiguous()
|
||||
active = torch.empty(len(contexts), dtype=torch.bool, device=contexts.device)
|
||||
_repeated_context_mask_kernel[(len(contexts),)](
|
||||
active,
|
||||
all_token_ids,
|
||||
all_token_ids.stride(0),
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
steps,
|
||||
steps.stride(0),
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
None,
|
||||
prior_contexts,
|
||||
prior_contexts.stride(0),
|
||||
prior_contexts.stride(1),
|
||||
steps,
|
||||
steps.stride(0),
|
||||
enabled,
|
||||
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
|
||||
CONTEXT_WIDTH=contexts.shape[1],
|
||||
MAX_HISTORY=0 if max_history is None else max_history,
|
||||
INCLUDE_PROMPT=include_prompt,
|
||||
SKIP_PARTIAL_CONTEXT=False,
|
||||
HAS_HISTORY_OFFSETS=True,
|
||||
SPECULATIVE_CONTEXTS=False,
|
||||
DRAFT_CONTEXTS=True,
|
||||
BLOCK=512,
|
||||
)
|
||||
return active
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mulhilo32(multiplier: tl.constexpr, value):
|
||||
high = tl_math.umulhi(multiplier, value)
|
||||
@@ -351,28 +592,58 @@ def _philox_gumbel_kernel(
|
||||
seeds_ptr,
|
||||
pos_ptr,
|
||||
temp_ptr,
|
||||
logits_cache_ptr,
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
key_0_value,
|
||||
key_1_value,
|
||||
vocab_size,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
IS_DRAFTING: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
block_index = tl.program_id(1)
|
||||
groups = block_index * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4)
|
||||
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = candidate < vocab_size
|
||||
|
||||
if skip_mask_ptr is not None:
|
||||
req_state_idx = row
|
||||
valid_req = True
|
||||
if expanded_idx_mapping_ptr is not None:
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
|
||||
valid_req = req_state_idx >= 0
|
||||
|
||||
if logits_cache_ptr is not None:
|
||||
if PER_TOKEN_COL:
|
||||
col = tl.load(logits_cache_col_ptr + row)
|
||||
else:
|
||||
col = tl.load(logits_cache_col_ptr)
|
||||
cached_logits = tl.load(
|
||||
logits_cache_source_ptr + row * logits_cache_source_stride + candidate,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
logits_cache_ptr
|
||||
+ req_state_idx * logits_cache_stride_0
|
||||
+ col * logits_cache_stride_1
|
||||
+ candidate,
|
||||
cached_logits,
|
||||
mask=mask & valid_req,
|
||||
)
|
||||
|
||||
if skip_mask_ptr is not None: # noqa: SIM102
|
||||
if tl.load(skip_mask_ptr + row):
|
||||
temp = tl.load(temp_ptr + req_state_idx, mask=valid_req, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + row)
|
||||
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = candidate < vocab_size
|
||||
logits_row = logits_ptr + row * logits_stride
|
||||
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
|
||||
tl.float32
|
||||
@@ -384,7 +655,7 @@ def _philox_gumbel_kernel(
|
||||
seed,
|
||||
pos,
|
||||
temp,
|
||||
IS_DRAFTING=False,
|
||||
IS_DRAFTING=IS_DRAFTING,
|
||||
USE_FP64=USE_FP64,
|
||||
APPLY_TEMPERATURE=False,
|
||||
)
|
||||
@@ -443,12 +714,37 @@ def philox_gumbel_sample(
|
||||
seeds: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
is_drafting: bool = False,
|
||||
logits_cache: torch.Tensor | None = None,
|
||||
logits_cache_col: torch.Tensor | None = None,
|
||||
logits_cache_source: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
sampling_state = (expanded_idx_mapping, temperatures, seeds, positions)
|
||||
if skip_mask is None and any(value is not None for value in sampling_state):
|
||||
raise ValueError("sampling state requires skip_mask")
|
||||
if skip_mask is not None and any(value is None for value in sampling_state):
|
||||
raise ValueError("skip_mask requires complete sampling state")
|
||||
if logits_cache is not None:
|
||||
if logits_cache_col is None or logits_cache_source is None:
|
||||
raise ValueError("logits cache requires its column and source logits")
|
||||
if expanded_idx_mapping is None:
|
||||
raise ValueError("logits cache requires expanded_idx_mapping")
|
||||
assert logits_cache_source.shape == logits.shape, (
|
||||
"logits cache source must match sampled logits shape"
|
||||
)
|
||||
assert logits_cache_source.device == logits.device, (
|
||||
"logits cache source must be on the sampled logits device"
|
||||
)
|
||||
assert logits_cache_source.dtype == logits_cache.dtype, (
|
||||
"logits cache source and destination must have the same dtype"
|
||||
)
|
||||
assert logits_cache.size(-1) >= logits.shape[-1], (
|
||||
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
|
||||
f"than the sampled logits ({logits.shape[-1]}). Cached logits would be "
|
||||
"truncated."
|
||||
)
|
||||
elif logits_cache_col is not None or logits_cache_source is not None:
|
||||
raise ValueError("logits cache column and source require logits_cache")
|
||||
|
||||
if logits.stride(-1) != 1:
|
||||
logits = logits.contiguous()
|
||||
@@ -460,6 +756,10 @@ def philox_gumbel_sample(
|
||||
expanded_idx_mapping = expanded_idx_mapping.contiguous()
|
||||
if positions is not None:
|
||||
positions = positions.contiguous()
|
||||
if logits_cache_col is not None:
|
||||
logits_cache_col = logits_cache_col.contiguous()
|
||||
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
|
||||
logits_cache_source = logits_cache_source.contiguous()
|
||||
num_tokens, vocab_size = logits.shape
|
||||
block_size = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, block_size)
|
||||
@@ -483,12 +783,20 @@ def philox_gumbel_sample(
|
||||
seeds,
|
||||
positions,
|
||||
temperatures,
|
||||
logits_cache,
|
||||
logits_cache.stride(0) if logits_cache is not None else 0,
|
||||
logits_cache.stride(1) if logits_cache is not None else 0,
|
||||
logits_cache_col,
|
||||
logits_cache_source,
|
||||
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
|
||||
key & _UINT32_MASK_VALUE,
|
||||
key >> 32,
|
||||
vocab_size,
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
BLOCK_SIZE=block_size,
|
||||
USE_FP64=use_fp64,
|
||||
IS_DRAFTING=is_drafting,
|
||||
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
|
||||
)
|
||||
max_block_index = local_max.argmax(dim=-1, keepdim=True)
|
||||
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
|
||||
|
||||
Reference in New Issue
Block a user