[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -48,6 +48,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
@@ -728,10 +729,13 @@ class DiffusionGemmaRequestStates:
|
||||
max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
def init_canvas(self, slot_indices_np: np.ndarray) -> None:
|
||||
"""Initialize canvas with random tokens for the given slots."""
|
||||
n = slot_indices_np.shape[0]
|
||||
self.canvas[slot_indices_np] = torch.randint(
|
||||
def init_canvas(self, slot_indices: torch.Tensor) -> None:
|
||||
"""Initialize canvas with random tokens for the given slots.
|
||||
|
||||
`slot_indices` must already be on device to avoid a cpu->gpu sync.
|
||||
"""
|
||||
n = slot_indices.shape[0]
|
||||
self.canvas[slot_indices] = torch.randint(
|
||||
0,
|
||||
self.vocab_size,
|
||||
(n, self.canvas_length),
|
||||
@@ -740,15 +744,15 @@ class DiffusionGemmaRequestStates:
|
||||
)
|
||||
|
||||
def add_request(self, slot_idx: int) -> None:
|
||||
self.is_encoder_phase[slot_idx] = True
|
||||
self.init_canvas(torch.tensor([slot_idx], device=self.device))
|
||||
self.step[slot_idx] = 0
|
||||
self.accepted_canvas_history_len[slot_idx] = 0
|
||||
self.is_encoder_phase[slot_idx].fill_(True)
|
||||
self.init_canvas(async_tensor_h2d([slot_idx], device=self.device))
|
||||
self.step[slot_idx].fill_(0)
|
||||
self.accepted_canvas_history_len[slot_idx].fill_(0)
|
||||
self.self_conditioning_embeds[slot_idx] = 0
|
||||
|
||||
def remove_request(self, slot_idx: int) -> None:
|
||||
self.is_encoder_phase[slot_idx] = False
|
||||
self.accepted_canvas_history_len[slot_idx] = 0
|
||||
self.is_encoder_phase[slot_idx].fill_(False)
|
||||
self.accepted_canvas_history_len[slot_idx].fill_(0)
|
||||
self.self_conditioning_embeds[slot_idx] = 0
|
||||
|
||||
|
||||
@@ -1148,11 +1152,15 @@ class DiffusionSampler:
|
||||
ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]]
|
||||
if len(ps) == 0:
|
||||
return
|
||||
states.init_canvas(ps)
|
||||
self.req_states.draft_tokens[ps, : self.canvas_length] = states.canvas[ps]
|
||||
# Move the slot indices across once, up front: indexing a device
|
||||
# tensor with a numpy array copies them over synchronously each time.
|
||||
ps_gpu = async_copy_to_gpu(
|
||||
ps.astype(np.int64), device=states.is_encoder_phase.device
|
||||
)
|
||||
states.init_canvas(ps_gpu)
|
||||
self.req_states.draft_tokens[ps_gpu, : self.canvas_length] = states.canvas[
|
||||
ps_gpu
|
||||
]
|
||||
states.is_encoder_phase.index_fill_(0, ps_gpu, False)
|
||||
|
||||
def _handle_prefill(
|
||||
|
||||
Reference in New Issue
Block a user