[Feature][DSpark]: Logprobs adaptive verification (#52242)
Signed-off-by: Naveenraj Kamalakannan <therealnaveenkamal@gmail.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
ca29cd48ba
commit
d5cadcee86
@@ -411,7 +411,7 @@ def _validate_logprobs(
|
||||
ref_prompt_logprob_toks,
|
||||
ref_prompt_logprob_vals,
|
||||
ref_prompt_token_ranks,
|
||||
_,
|
||||
*_,
|
||||
) = ref_prompt_logprobs
|
||||
for idx, (prompt_token, pos_logprob_dict) in enumerate(
|
||||
zip(prompt_token_ids[1:], prompt_logprobs[1:])
|
||||
|
||||
@@ -42,6 +42,25 @@ def test_logprobs_tensors_cat():
|
||||
assert LogprobsTensors.cat([first]) is first
|
||||
|
||||
|
||||
def test_logprobs_tensors_tolists_with_tensor_boundaries():
|
||||
"""Adaptive verification hands over the request boundaries as a tensor
|
||||
(they only exist on device); tolists() must materialize it as a plain
|
||||
list so slice_request splits requests correctly."""
|
||||
tensors = LogprobsTensors(
|
||||
torch.tensor([[1, 2], [3, 4], [5, 6]]),
|
||||
torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]),
|
||||
torch.tensor([1, 2, 3]),
|
||||
cu_num_generated_tokens_tensor=torch.tensor([0, 2, 3], dtype=torch.int32),
|
||||
)
|
||||
|
||||
lists = tensors.to_cpu_nonblocking().tolists()
|
||||
|
||||
assert lists.cu_num_generated_tokens == [0, 2, 3]
|
||||
sliced = lists.slice_request(1, 1)
|
||||
assert sliced.logprob_token_ids.tolist() == [[5, 6]]
|
||||
assert sliced.sampled_token_ranks.tolist() == [3]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_tolist():
|
||||
tensors = SamplingMaskTensors(
|
||||
packed_mask=torch.tensor(
|
||||
|
||||
@@ -51,6 +51,7 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
|
||||
rejection_sampler.num_speculative_steps = 3
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
|
||||
def fake_verify(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user