[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-19 10:35:54 -04:00
committed by GitHub
parent a78b842d0e
commit b82e908b4c
23 changed files with 129 additions and 108 deletions
+7 -15
View File
@@ -30,10 +30,8 @@ from vllm.model_executor.models.interfaces import supports_any_eagle
from vllm.multimodal import NestedTensors
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import (
is_pin_memory_available,
)
from vllm.utils.torch_utils import (
async_tensor_h2d,
direct_register_custom_op,
)
@@ -498,10 +496,9 @@ def isin_list(
elements: torch.Tensor,
test_elements_list: list[int],
) -> torch.Tensor:
test_elements = torch.tensor(
test_elements_list,
pin_memory=is_pin_memory_available(),
).to(device=elements.device, non_blocking=True)
test_elements = async_tensor_h2d(
test_elements_list, dtype=torch.int64, device=elements.device
)
return torch.isin(elements, test_elements)
@@ -772,14 +769,9 @@ def extract_layer_index(layer_name: str, num_attn_module: int = 1) -> int:
return layer_index
def cast_overflow_tensors(
tensors: torch.Tensor,
offset: float = 1000,
) -> torch.Tensor:
if tensors.isinf().any() or tensors.isnan().any():
clamp_value = torch.finfo(tensors.dtype).max - offset
tensors = torch.clamp(tensors, min=-clamp_value, max=clamp_value)
return tensors
def cast_overflow_tensors(tensors: torch.Tensor, offset: float = 1000) -> torch.Tensor:
clamp_value = torch.finfo(tensors.dtype).max - offset
return torch.clamp(tensors, min=-clamp_value, max=clamp_value)
def fast_topk(