[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: vx120 <893600387@qq.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
vx120
parent
d6bce42983
commit
560ef78bfe
@@ -4,6 +4,7 @@ import dataclasses
|
||||
from concurrent.futures import Future
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -43,6 +44,7 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
SamplingMaskLists,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
@@ -667,6 +669,51 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
|
||||
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
|
||||
|
||||
|
||||
def test_update_from_output_routes_sampling_masks_by_request():
|
||||
"""Each request receives the sampler row at its own batch index."""
|
||||
scheduler = create_scheduler()
|
||||
scheduler.return_sampling_mask = True
|
||||
requests = create_requests(num_requests=3, max_tokens=10)
|
||||
for req in requests:
|
||||
req.num_computed_tokens = req.num_tokens
|
||||
scheduler.requests[req.request_id] = req
|
||||
scheduler.running.append(req)
|
||||
req.status = RequestStatus.RUNNING
|
||||
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens={req.request_id: 1 for req in requests},
|
||||
total_num_scheduled_tokens=3,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=[req.request_id for req in requests],
|
||||
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
|
||||
sampled_token_ids=[[1], [3], [4]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
sampling_masks=SamplingMaskLists(
|
||||
np.array([1, 2, 3, 4, 5, 6], dtype=np.int32), np.array([0, 2, 3, 6])
|
||||
),
|
||||
)
|
||||
|
||||
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
|
||||
|
||||
assert [out.request_id for out in outputs] == [req.request_id for req in requests]
|
||||
assert [out.new_sampling_mask.token_ids.tolist() for out in outputs] == [
|
||||
[1, 2],
|
||||
[3],
|
||||
[4, 5, 6],
|
||||
]
|
||||
assert all(out.new_sampling_mask.offsets is None for out in outputs)
|
||||
|
||||
|
||||
def test_stop_via_update_from_output():
|
||||
"""Test stopping behavior through update_from_output"""
|
||||
scheduler = create_scheduler(num_speculative_tokens=1)
|
||||
|
||||
Reference in New Issue
Block a user