[Refactor] Remove dead code multiple places (#61843)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -353,46 +353,6 @@ def test_silu_and_mul_with_clamp(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_masked_situ_and_mul(
|
||||
default_vllm_config,
|
||||
linear_beta: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Masked SITU computes valid expert rows and preserves padded zeros."""
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts, max_num_tokens, d = 4, 7, 512
|
||||
beta = 1.5
|
||||
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
|
||||
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
|
||||
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
|
||||
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
output, input, expert_num_tokens, beta, linear_beta
|
||||
)
|
||||
|
||||
gate, up = input.float().chunk(2, dim=-1)
|
||||
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
if linear_beta > 0:
|
||||
up = linear_beta * torch.tanh(up / linear_beta)
|
||||
expected = (expected * up).to(dtype)
|
||||
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
|
||||
torch.testing.assert_close(
|
||||
output[expert, :num_tokens],
|
||||
expected[expert, :num_tokens],
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.masked_situ_and_mul,
|
||||
(output, input, expert_num_tokens, beta, linear_beta),
|
||||
)
|
||||
|
||||
|
||||
MOE_ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
|
||||
pytest.param(
|
||||
|
||||
@@ -10,7 +10,6 @@ import torch.nn.functional as F
|
||||
import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.layers.mhc as mhc_layers
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
_torch_hc_prenorm_gemm,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_delayed_tilelang,
|
||||
)
|
||||
@@ -1002,6 +1001,19 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
|
||||
torch.testing.assert_close(actual, expected, atol=5e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _torch_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
) -> None:
|
||||
assert out.shape[0] == 1
|
||||
assert sqrsum.shape[0] == 1
|
||||
x_float = x.float()
|
||||
out[0].copy_(x_float @ fn.t())
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TILELANG_MHC,
|
||||
reason="TileLang MHC support required",
|
||||
|
||||
Reference in New Issue
Block a user