[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -248,13 +248,13 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
|||||||
feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
|
feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
|
||||||
"audio", chunk_counts, dim=0
|
"audio", chunk_counts, dim=0
|
||||||
),
|
),
|
||||||
chunk_counts=MultiModalFieldConfig.batched("audio"),
|
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||||
)
|
)
|
||||||
return dict(
|
return dict(
|
||||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||||
input_features=MultiModalFieldConfig.batched("audio"),
|
input_features=MultiModalFieldConfig.batched("audio"),
|
||||||
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
|
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
|
||||||
chunk_counts=MultiModalFieldConfig.batched("audio"),
|
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
|
|||||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.utils.torch_utils import async_tensor_h2d
|
||||||
from vllm.v1.outputs import LogprobsTensors
|
from vllm.v1.outputs import LogprobsTensors
|
||||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||||
@@ -728,10 +729,13 @@ class DiffusionGemmaRequestStates:
|
|||||||
max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device
|
max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device
|
||||||
)
|
)
|
||||||
|
|
||||||
def init_canvas(self, slot_indices_np: np.ndarray) -> None:
|
def init_canvas(self, slot_indices: torch.Tensor) -> None:
|
||||||
"""Initialize canvas with random tokens for the given slots."""
|
"""Initialize canvas with random tokens for the given slots.
|
||||||
n = slot_indices_np.shape[0]
|
|
||||||
self.canvas[slot_indices_np] = torch.randint(
|
`slot_indices` must already be on device to avoid a cpu->gpu sync.
|
||||||
|
"""
|
||||||
|
n = slot_indices.shape[0]
|
||||||
|
self.canvas[slot_indices] = torch.randint(
|
||||||
0,
|
0,
|
||||||
self.vocab_size,
|
self.vocab_size,
|
||||||
(n, self.canvas_length),
|
(n, self.canvas_length),
|
||||||
@@ -740,15 +744,15 @@ class DiffusionGemmaRequestStates:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def add_request(self, slot_idx: int) -> None:
|
def add_request(self, slot_idx: int) -> None:
|
||||||
self.is_encoder_phase[slot_idx] = True
|
self.is_encoder_phase[slot_idx].fill_(True)
|
||||||
self.init_canvas(torch.tensor([slot_idx], device=self.device))
|
self.init_canvas(async_tensor_h2d([slot_idx], device=self.device))
|
||||||
self.step[slot_idx] = 0
|
self.step[slot_idx].fill_(0)
|
||||||
self.accepted_canvas_history_len[slot_idx] = 0
|
self.accepted_canvas_history_len[slot_idx].fill_(0)
|
||||||
self.self_conditioning_embeds[slot_idx] = 0
|
self.self_conditioning_embeds[slot_idx] = 0
|
||||||
|
|
||||||
def remove_request(self, slot_idx: int) -> None:
|
def remove_request(self, slot_idx: int) -> None:
|
||||||
self.is_encoder_phase[slot_idx] = False
|
self.is_encoder_phase[slot_idx].fill_(False)
|
||||||
self.accepted_canvas_history_len[slot_idx] = 0
|
self.accepted_canvas_history_len[slot_idx].fill_(0)
|
||||||
self.self_conditioning_embeds[slot_idx] = 0
|
self.self_conditioning_embeds[slot_idx] = 0
|
||||||
|
|
||||||
|
|
||||||
@@ -1148,11 +1152,15 @@ class DiffusionSampler:
|
|||||||
ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]]
|
ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]]
|
||||||
if len(ps) == 0:
|
if len(ps) == 0:
|
||||||
return
|
return
|
||||||
states.init_canvas(ps)
|
# Move the slot indices across once, up front: indexing a device
|
||||||
self.req_states.draft_tokens[ps, : self.canvas_length] = states.canvas[ps]
|
# tensor with a numpy array copies them over synchronously each time.
|
||||||
ps_gpu = async_copy_to_gpu(
|
ps_gpu = async_copy_to_gpu(
|
||||||
ps.astype(np.int64), device=states.is_encoder_phase.device
|
ps.astype(np.int64), device=states.is_encoder_phase.device
|
||||||
)
|
)
|
||||||
|
states.init_canvas(ps_gpu)
|
||||||
|
self.req_states.draft_tokens[ps_gpu, : self.canvas_length] = states.canvas[
|
||||||
|
ps_gpu
|
||||||
|
]
|
||||||
states.is_encoder_phase.index_fill_(0, ps_gpu, False)
|
states.is_encoder_phase.index_fill_(0, ps_gpu, False)
|
||||||
|
|
||||||
def _handle_prefill(
|
def _handle_prefill(
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
|
|||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||||
|
from vllm.utils.torch_utils import async_tensor_h2d
|
||||||
|
|
||||||
from .interfaces import (
|
from .interfaces import (
|
||||||
MultiModalEmbeddings,
|
MultiModalEmbeddings,
|
||||||
@@ -2039,7 +2040,20 @@ class Gemma4ForConditionalGeneration(
|
|||||||
) -> torch.Tensor | None:
|
) -> torch.Tensor | None:
|
||||||
logits = self.language_model.compute_logits(hidden_states)
|
logits = self.language_model.compute_logits(hidden_states)
|
||||||
if logits is not None and self._suppress_token_ids:
|
if logits is not None and self._suppress_token_ids:
|
||||||
logits[:, self._suppress_token_ids] = -float("inf")
|
# Cache a per-device index tensor for the (static) suppressed-token
|
||||||
|
# set and use `index_fill_`, so neither the Python-list indices nor
|
||||||
|
# the scalar fill value take a host roundtrip per call.
|
||||||
|
cache = getattr(self, "_suppress_token_ids_cache", None)
|
||||||
|
if cache is None:
|
||||||
|
cache = {}
|
||||||
|
self._suppress_token_ids_cache = cache
|
||||||
|
suppress_idx = cache.get(logits.device)
|
||||||
|
if suppress_idx is None:
|
||||||
|
suppress_idx = async_tensor_h2d(
|
||||||
|
self._suppress_token_ids, dtype=torch.long, device=logits.device
|
||||||
|
)
|
||||||
|
cache[logits.device] = suppress_idx
|
||||||
|
logits.index_fill_(1, suppress_idx, -float("inf"))
|
||||||
return logits
|
return logits
|
||||||
|
|
||||||
# ------------------------------------------------------------------ #
|
# ------------------------------------------------------------------ #
|
||||||
|
|||||||
@@ -548,9 +548,9 @@ class Glm4vVisionEmbeddings(nn.Module):
|
|||||||
else:
|
else:
|
||||||
# Convert inputs to tensors if needed
|
# Convert inputs to tensors if needed
|
||||||
if isinstance(lengths, list):
|
if isinstance(lengths, list):
|
||||||
lengths = torch.tensor(lengths, device=device, dtype=torch.long)
|
lengths = async_tensor_h2d(lengths, device=device, dtype=torch.long)
|
||||||
if not isinstance(image_shapes, torch.Tensor):
|
if not isinstance(image_shapes, torch.Tensor):
|
||||||
image_shapes = torch.tensor(
|
image_shapes = async_tensor_h2d(
|
||||||
image_shapes, device=device, dtype=torch.long
|
image_shapes, device=device, dtype=torch.long
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -426,12 +426,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
|||||||
),
|
),
|
||||||
image_num_patches=MultiModalFieldConfig.batched("image"),
|
image_num_patches=MultiModalFieldConfig.batched("image"),
|
||||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
image_embeds=MultiModalFieldConfig.batched("image"),
|
||||||
image_token_id=MultiModalFieldConfig.shared("image", num_images),
|
image_token_id=MultiModalFieldConfig.shared(
|
||||||
|
"image", num_images, keep_on_cpu=True
|
||||||
|
),
|
||||||
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
||||||
"video", video_num_patches
|
"video", video_num_patches
|
||||||
),
|
),
|
||||||
video_num_patches=MultiModalFieldConfig.batched("video"),
|
video_num_patches=MultiModalFieldConfig.batched("video"),
|
||||||
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
|
video_token_id=MultiModalFieldConfig.shared(
|
||||||
|
"video", num_videos, keep_on_cpu=True
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
|
|||||||
@@ -478,7 +478,9 @@ class InternVLMultiModalProcessor(
|
|||||||
"video", video_num_patches
|
"video", video_num_patches
|
||||||
),
|
),
|
||||||
video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||||
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
|
video_token_id=MultiModalFieldConfig.shared(
|
||||||
|
"video", num_videos, keep_on_cpu=True
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
|
|||||||
@@ -871,12 +871,12 @@ def _keye_field_config(
|
|||||||
return dict(
|
return dict(
|
||||||
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
||||||
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
||||||
image_grid_thw=MultiModalFieldConfig.batched("image"),
|
image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||||
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
||||||
"video", video_grid_sizes
|
"video", video_grid_sizes
|
||||||
),
|
),
|
||||||
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
|
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
|
||||||
video_grid_thw=MultiModalFieldConfig.batched("video"),
|
video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -1291,8 +1291,9 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
|||||||
image_grid_thw = image_input["image_grid_thw"]
|
image_grid_thw = image_input["image_grid_thw"]
|
||||||
assert image_grid_thw.ndim == 2
|
assert image_grid_thw.ndim == 2
|
||||||
|
|
||||||
for idx, thaw in enumerate(image_grid_thw):
|
image_grid_thw_list = image_grid_thw.tolist()
|
||||||
thw_tuple = tuple(thaw.detach().cpu().numpy().tolist())
|
for idx, thw in enumerate(image_grid_thw_list):
|
||||||
|
thw_tuple = tuple(thw)
|
||||||
numel = np.prod(thw_tuple)
|
numel = np.prod(thw_tuple)
|
||||||
image_grid_hws.append(thw_tuple)
|
image_grid_hws.append(thw_tuple)
|
||||||
image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
|
image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
|
||||||
|
|||||||
@@ -266,9 +266,10 @@ class Phi3HDImageEmbedding(nn.Module):
|
|||||||
)
|
)
|
||||||
|
|
||||||
batch_image_features_proj = []
|
batch_image_features_proj = []
|
||||||
|
image_sizes_list = image_sizes.tolist()
|
||||||
# need a for loop to process each image because of different image sizes
|
# need a for loop to process each image because of different image sizes
|
||||||
# (patch arrangement is different for each image)
|
# (patch arrangement is different for each image)
|
||||||
for i, img_size in enumerate(image_sizes):
|
for i, img_size in enumerate(image_sizes_list):
|
||||||
h, w = img_size
|
h, w = img_size
|
||||||
h_crop = h // 336
|
h_crop = h // 336
|
||||||
w_crop = w // 336
|
w_crop = w // 336
|
||||||
@@ -426,7 +427,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
|||||||
) -> Mapping[str, MultiModalFieldConfig]:
|
) -> Mapping[str, MultiModalFieldConfig]:
|
||||||
return dict(
|
return dict(
|
||||||
pixel_values=MultiModalFieldConfig.batched("image"),
|
pixel_values=MultiModalFieldConfig.batched("image"),
|
||||||
image_sizes=MultiModalFieldConfig.batched("image"),
|
image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
image_embeds=MultiModalFieldConfig.batched("image"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -270,8 +270,7 @@ class Phi4MMImageEncoder(nn.Module):
|
|||||||
pixel_values = pixel_values.flatten(0, 1)
|
pixel_values = pixel_values.flatten(0, 1)
|
||||||
|
|
||||||
img_features = self.get_img_features(
|
img_features = self.get_img_features(
|
||||||
pixel_values,
|
pixel_values, image_attention_mask.bool().flatten(0, 1)
|
||||||
image_attention_mask.type(torch.BoolTensor).flatten(0, 1).to(target_device),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
base_feat_height_target = self.base_feat_height_target
|
base_feat_height_target = self.base_feat_height_target
|
||||||
|
|||||||
@@ -200,7 +200,7 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo])
|
|||||||
patch_pixel_values=MultiModalFieldConfig.flat_from_sizes(
|
patch_pixel_values=MultiModalFieldConfig.flat_from_sizes(
|
||||||
"image", num_patches
|
"image", num_patches
|
||||||
),
|
),
|
||||||
num_patches=MultiModalFieldConfig.batched("image"),
|
num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||||
patch_newline_mask=MultiModalFieldConfig.flat_from_sizes(
|
patch_newline_mask=MultiModalFieldConfig.flat_from_sizes(
|
||||||
"image", num_patches
|
"image", num_patches
|
||||||
),
|
),
|
||||||
@@ -682,7 +682,8 @@ class Step3VLForConditionalGeneration(
|
|||||||
|
|
||||||
merged_image_features = []
|
merged_image_features = []
|
||||||
cur_patch_idx = 0
|
cur_patch_idx = 0
|
||||||
for i, num_patch in enumerate(num_patches):
|
num_patches_list = num_patches.tolist()
|
||||||
|
for i, num_patch in enumerate(num_patches_list):
|
||||||
cur_feature = []
|
cur_feature = []
|
||||||
if num_patch > 0:
|
if num_patch > 0:
|
||||||
patch_slice = patch_image_features[
|
patch_slice = patch_image_features[
|
||||||
|
|||||||
@@ -442,6 +442,7 @@ class CommonAttentionMetadata:
|
|||||||
# Needed by FastPrefillAttentionBuilder
|
# Needed by FastPrefillAttentionBuilder
|
||||||
logits_indices_padded: torch.Tensor | None = None
|
logits_indices_padded: torch.Tensor | None = None
|
||||||
num_logits_indices: int | None = None
|
num_logits_indices: int | None = None
|
||||||
|
max_logits_per_req: int | None = None
|
||||||
|
|
||||||
# Needed by CrossAttentionBuilder
|
# Needed by CrossAttentionBuilder
|
||||||
encoder_seq_lens: torch.Tensor | None = None
|
encoder_seq_lens: torch.Tensor | None = None
|
||||||
@@ -590,6 +591,7 @@ class CommonAttentionMetadata:
|
|||||||
else self.causal,
|
else self.causal,
|
||||||
logits_indices_padded=self.logits_indices_padded,
|
logits_indices_padded=self.logits_indices_padded,
|
||||||
num_logits_indices=self.num_logits_indices,
|
num_logits_indices=self.num_logits_indices,
|
||||||
|
max_logits_per_req=self.max_logits_per_req,
|
||||||
encoder_seq_lens=maybe_slice_reqs(self.encoder_seq_lens),
|
encoder_seq_lens=maybe_slice_reqs(self.encoder_seq_lens),
|
||||||
encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu),
|
encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu),
|
||||||
dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens),
|
dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens),
|
||||||
|
|||||||
@@ -150,11 +150,15 @@ class Mamba2AttentionMetadataBuilder(
|
|||||||
|
|
||||||
# Compute seq_idx for prefill only
|
# Compute seq_idx for prefill only
|
||||||
if common.num_prefills > 0:
|
if common.num_prefills > 0:
|
||||||
prep_initial_states = (
|
prep_initial_states = False
|
||||||
torch.any(common.has_initial_states_p).item()
|
if common.has_initial_states_p is not None:
|
||||||
if common.has_initial_states_p is not None
|
# Same condition as `has_initial_states_p`, but derived from CPU
|
||||||
else False
|
# data so it needs no D2H. `seq_lens_cpu_upper_bound` is precise
|
||||||
)
|
# for prefill rows, which is all this slice covers.
|
||||||
|
num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata(
|
||||||
|
common, common_attn_metadata
|
||||||
|
)
|
||||||
|
prep_initial_states = bool((num_computed_tokens_p_cpu > 0).any())
|
||||||
|
|
||||||
cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p = (
|
cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p = (
|
||||||
self._build_chunk_metadata_tensors(
|
self._build_chunk_metadata_tensors(
|
||||||
|
|||||||
@@ -324,6 +324,33 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
|||||||
|
|
||||||
return cu_chunk_seqlen, seq_idx, last_chunk_indices
|
return cu_chunk_seqlen, seq_idx, last_chunk_indices
|
||||||
|
|
||||||
|
def _prefill_cpu_metadata(
|
||||||
|
self,
|
||||||
|
common: M,
|
||||||
|
common_attn_metadata: CommonAttentionMetadata,
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
"""Prefill context lengths and query offsets, from CPU data only.
|
||||||
|
|
||||||
|
`seq_lens_cpu_upper_bound` is precise for prefill rows in all modes
|
||||||
|
(including async spec decode), so this avoids the D2H sync that
|
||||||
|
`compute_num_computed_tokens().cpu()` would force.
|
||||||
|
|
||||||
|
Returns (num_computed_tokens_p_cpu, query_start_loc_p_cpu).
|
||||||
|
"""
|
||||||
|
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||||
|
assert seq_lens_cpu is not None
|
||||||
|
num_reqs = common.num_reqs
|
||||||
|
num_prefills = common.num_prefills
|
||||||
|
query_start_loc_p_cpu = (
|
||||||
|
common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :]
|
||||||
|
- common.num_decode_tokens
|
||||||
|
)
|
||||||
|
prefill_query_lens_cpu = query_start_loc_p_cpu[1:] - query_start_loc_p_cpu[:-1]
|
||||||
|
num_computed_tokens_p_cpu = (
|
||||||
|
seq_lens_cpu[num_reqs - num_prefills : num_reqs] - prefill_query_lens_cpu
|
||||||
|
)
|
||||||
|
return num_computed_tokens_p_cpu, query_start_loc_p_cpu
|
||||||
|
|
||||||
def _build_chunk_metadata_tensors(
|
def _build_chunk_metadata_tensors(
|
||||||
self,
|
self,
|
||||||
chunk_size: int,
|
chunk_size: int,
|
||||||
@@ -334,23 +361,10 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
|||||||
Compute chunk metadata and return as device tensors.
|
Compute chunk metadata and return as device tensors.
|
||||||
Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p).
|
Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p).
|
||||||
"""
|
"""
|
||||||
num_reqs = common.num_reqs
|
|
||||||
num_prefills = common.num_prefills
|
num_prefills = common.num_prefills
|
||||||
num_decode_tokens = common.num_decode_tokens
|
|
||||||
|
|
||||||
# Derive prefill context lengths from CPU data only.
|
num_computed_tokens_p_cpu, query_start_loc_p_cpu = self._prefill_cpu_metadata(
|
||||||
# `seq_lens_cpu_upper_bound` is precise for prefill rows in all modes
|
common, common_attn_metadata
|
||||||
# (including async spec decode), so this avoids the D2H sync that
|
|
||||||
# `compute_num_computed_tokens().cpu()` would force.
|
|
||||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
|
||||||
assert seq_lens_cpu is not None
|
|
||||||
query_start_loc_p_cpu = (
|
|
||||||
common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :]
|
|
||||||
- num_decode_tokens
|
|
||||||
)
|
|
||||||
prefill_query_lens_cpu = query_start_loc_p_cpu[1:] - query_start_loc_p_cpu[:-1]
|
|
||||||
num_computed_tokens_p_cpu = (
|
|
||||||
seq_lens_cpu[num_reqs - num_prefills : num_reqs] - prefill_query_lens_cpu
|
|
||||||
)
|
)
|
||||||
|
|
||||||
cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata(
|
cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata(
|
||||||
|
|||||||
@@ -539,8 +539,14 @@ def make_kv_sharing_fast_prefill_common_attn_metadata(
|
|||||||
|
|
||||||
decode_query_start_loc[:1].fill_(0) # Avoid sync from scalar assignment.
|
decode_query_start_loc[:1].fill_(0) # Avoid sync from scalar assignment.
|
||||||
decode_query_start_loc[1:] = torch.cumsum(num_decode_tokens, dim=0)
|
decode_query_start_loc[1:] = torch.cumsum(num_decode_tokens, dim=0)
|
||||||
decode_max_query_len = int(num_decode_tokens.max().item())
|
|
||||||
total_num_decode_tokens = int(num_decode_tokens.sum().item())
|
# `num_decode_tokens` is a histogram over `logits_indices`, so its total is
|
||||||
|
# just how many there were -- already known as a Python int.
|
||||||
|
total_num_decode_tokens = num_logits_indices
|
||||||
|
|
||||||
|
# Largest per-request logits count.
|
||||||
|
decode_max_query_len = common_attn_metadata.max_logits_per_req
|
||||||
|
assert decode_max_query_len is not None
|
||||||
|
|
||||||
common_attn_metadata = CommonAttentionMetadata(
|
common_attn_metadata = CommonAttentionMetadata(
|
||||||
query_start_loc=decode_query_start_loc,
|
query_start_loc=decode_query_start_loc,
|
||||||
|
|||||||
@@ -2009,10 +2009,11 @@ class GPUModelRunner(
|
|||||||
) -> tuple[
|
) -> tuple[
|
||||||
torch.Tensor,
|
torch.Tensor,
|
||||||
SpecDecodeMetadata | None,
|
SpecDecodeMetadata | None,
|
||||||
|
int,
|
||||||
]:
|
]:
|
||||||
"""
|
"""
|
||||||
Returns:
|
Returns:
|
||||||
tuple[logits_indices, spec_decode_metadata]
|
tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens]
|
||||||
"""
|
"""
|
||||||
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||||
assert total_num_scheduled_tokens > 0
|
assert total_num_scheduled_tokens > 0
|
||||||
@@ -2324,6 +2325,7 @@ class GPUModelRunner(
|
|||||||
return (
|
return (
|
||||||
logits_indices,
|
logits_indices,
|
||||||
spec_decode_metadata,
|
spec_decode_metadata,
|
||||||
|
int(num_sampled_tokens.max()),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _build_attention_metadata(
|
def _build_attention_metadata(
|
||||||
@@ -2335,6 +2337,7 @@ class GPUModelRunner(
|
|||||||
num_reqs_padded: int | None = None,
|
num_reqs_padded: int | None = None,
|
||||||
ubatch_slices: UBatchSlices | None = None,
|
ubatch_slices: UBatchSlices | None = None,
|
||||||
logits_indices: torch.Tensor | None = None,
|
logits_indices: torch.Tensor | None = None,
|
||||||
|
max_num_sampled_tokens: int | None = None,
|
||||||
use_spec_decode: bool = False,
|
use_spec_decode: bool = False,
|
||||||
for_cudagraph_capture: bool = False,
|
for_cudagraph_capture: bool = False,
|
||||||
num_scheduled_tokens: dict[str, int] | None = None,
|
num_scheduled_tokens: dict[str, int] | None = None,
|
||||||
@@ -2509,6 +2512,7 @@ class GPUModelRunner(
|
|||||||
|
|
||||||
if logits_indices is not None and self.cache_config.kv_sharing_fast_prefill:
|
if logits_indices is not None and self.cache_config.kv_sharing_fast_prefill:
|
||||||
cm_base.num_logits_indices = logits_indices.size(0)
|
cm_base.num_logits_indices = logits_indices.size(0)
|
||||||
|
cm_base.max_logits_per_req = max_num_sampled_tokens
|
||||||
cm_base.logits_indices_padded = self._prepare_kv_sharing_fast_prefill(
|
cm_base.logits_indices_padded = self._prepare_kv_sharing_fast_prefill(
|
||||||
logits_indices
|
logits_indices
|
||||||
)
|
)
|
||||||
@@ -4341,9 +4345,8 @@ class GPUModelRunner(
|
|||||||
max_num_scheduled_tokens = int(num_scheduled_tokens_np.max())
|
max_num_scheduled_tokens = int(num_scheduled_tokens_np.max())
|
||||||
num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens
|
num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens
|
||||||
|
|
||||||
logits_indices, spec_decode_metadata = self._prepare_inputs(
|
logits_indices, spec_decode_metadata, max_num_sampled_tokens = (
|
||||||
scheduler_output,
|
self._prepare_inputs(scheduler_output, num_scheduled_tokens_np)
|
||||||
num_scheduled_tokens_np,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
cascade_attn_prefix_lens = None
|
cascade_attn_prefix_lens = None
|
||||||
@@ -4481,6 +4484,7 @@ class GPUModelRunner(
|
|||||||
max_query_len=max_num_scheduled_tokens,
|
max_query_len=max_num_scheduled_tokens,
|
||||||
ubatch_slices=ubatch_slices_attn,
|
ubatch_slices=ubatch_slices_attn,
|
||||||
logits_indices=logits_indices,
|
logits_indices=logits_indices,
|
||||||
|
max_num_sampled_tokens=max_num_sampled_tokens,
|
||||||
use_spec_decode=use_spec_decode,
|
use_spec_decode=use_spec_decode,
|
||||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||||
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
||||||
|
|||||||
Reference in New Issue
Block a user