[MRV2] Buffer util simplifications (#56888)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -269,11 +269,11 @@ from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
async_tensor_h2d,
|
||||
direct_register_custom_op,
|
||||
get_dtype_size,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
np_to_pinned_tensor,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
@@ -1980,11 +1980,6 @@ def plan_mla_context_chunks(
|
||||
return plans
|
||||
|
||||
|
||||
def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
|
||||
"""Pinned int32 CPU tensor backing one concatenated per-chunk field."""
|
||||
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
|
||||
|
||||
|
||||
def align_mla_chunked_context_workspace_size(
|
||||
vllm_config: VllmConfig,
|
||||
workspace_size: int,
|
||||
@@ -2161,24 +2156,28 @@ def build_mla_chunked_context_metadata(
|
||||
token_offset += num_tokens
|
||||
local_token_offset += num_local_tokens
|
||||
|
||||
seq_lens_cpu = _flat_int32(seq_lens_flat)
|
||||
starts_and_context_lens = _flat_int32(starts_flat + context_lens).to(
|
||||
device, non_blocking=True
|
||||
seq_lens_cpu = torch.tensor(
|
||||
seq_lens_flat, dtype=torch.int32, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
starts_and_context_lens = async_tensor_h2d(
|
||||
starts_flat + context_lens, device=device, dtype=torch.int32
|
||||
)
|
||||
starts = starts_and_context_lens[: len(starts_flat)]
|
||||
context_lens_gpu = starts_and_context_lens[len(starts_flat) :]
|
||||
cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True)
|
||||
cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True)
|
||||
token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to(
|
||||
device, non_blocking=True
|
||||
cu_seq_lens = async_tensor_h2d(cu_seq_lens_flat, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = async_tensor_h2d(cu_seqlens_q_flat, device=device, dtype=torch.int32)
|
||||
token_to_seq = async_tensor_h2d(
|
||||
np.concatenate(token_to_seq_parts), device=device, dtype=torch.int32
|
||||
)
|
||||
if use_dcp:
|
||||
padded_local_cu_seq_lens = _flat_int32(padded_local_cu_seq_lens_flat).to(
|
||||
device, non_blocking=True
|
||||
padded_local_cu_seq_lens = async_tensor_h2d(
|
||||
padded_local_cu_seq_lens_flat, device=device, dtype=torch.int32
|
||||
)
|
||||
padded_local_token_to_seq = async_tensor_h2d(
|
||||
np.concatenate(padded_local_token_to_seq_parts),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
padded_local_token_to_seq = _flat_int32(
|
||||
np.concatenate(padded_local_token_to_seq_parts)
|
||||
).to(device, non_blocking=True)
|
||||
|
||||
chunks: list[MLACommonPrefillMetadata.ContextChunk] = []
|
||||
for index, (plan, layout) in enumerate(zip(plans, layouts)):
|
||||
|
||||
@@ -53,7 +53,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
|
||||
@@ -1159,7 +1159,7 @@ class DiffusionSampler:
|
||||
return
|
||||
# Move the slot indices across once, up front: indexing a device
|
||||
# tensor with a numpy array copies them over synchronously each time.
|
||||
ps_gpu = async_copy_to_gpu(
|
||||
ps_gpu = async_tensor_h2d(
|
||||
ps.astype(np.int64), device=states.is_encoder_phase.device
|
||||
)
|
||||
states.init_canvas(ps_gpu)
|
||||
@@ -1266,7 +1266,7 @@ class DiffusionSampler:
|
||||
# was truncated near max_model_len, in which case the scheduler gave us
|
||||
# fewer than CL logits for that request.
|
||||
valid_canvas_len_np = per_req_nlogits_np[per_req_nlogits_np > 0]
|
||||
valid_canvas_len = async_copy_to_gpu(
|
||||
valid_canvas_len = async_tensor_h2d(
|
||||
valid_canvas_len_np.astype(np.int64), device=device
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user