Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -17,6 +17,7 @@ import vllm.config as vllm_config_module
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils
|
||||
from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
@@ -140,10 +141,15 @@ def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]:
|
||||
return quantization_config
|
||||
|
||||
|
||||
def _stub_quant_config() -> QuantizationConfig:
|
||||
return cast(QuantizationConfig, object())
|
||||
|
||||
|
||||
def get_fse_test_model_config(
|
||||
model_type: str,
|
||||
quantization_config: dict[str, Any],
|
||||
) -> tuple[object, type[nn.Module]]:
|
||||
config: object
|
||||
if model_type == "minimax_m3":
|
||||
config = MiniMaxM3TextConfig(
|
||||
hidden_size=128,
|
||||
@@ -301,7 +307,7 @@ def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled(
|
||||
)
|
||||
|
||||
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
object(), "model.layers.0.mlp"
|
||||
_stub_quant_config(), "model.layers.0.mlp"
|
||||
)
|
||||
|
||||
|
||||
@@ -316,7 +322,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
|
||||
|
||||
assert (
|
||||
fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
object(), "model.layers.0.mlp"
|
||||
_stub_quant_config(), "model.layers.0.mlp"
|
||||
)
|
||||
is False
|
||||
)
|
||||
@@ -325,7 +331,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
|
||||
def test_resolve_layer_fused_shared_expert_passes_module_prefixes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
quant_config = object()
|
||||
quant_config = _stub_quant_config()
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils.rocm_aiter_ops,
|
||||
"is_fusion_moe_shared_experts_enabled",
|
||||
@@ -374,7 +380,7 @@ def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization(
|
||||
)
|
||||
|
||||
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
object(), "model.layers.0.mlp"
|
||||
_stub_quant_config(), "model.layers.0.mlp"
|
||||
)
|
||||
assert "shared experts are excluded" in caplog.text
|
||||
|
||||
@@ -1214,7 +1220,7 @@ def test_quark_packed_layer_config_must_match_global_config() -> None:
|
||||
|
||||
def test_non_quark_shared_expert_fse_is_incompatible() -> None:
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
object(),
|
||||
_stub_quant_config(),
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_experts",
|
||||
)
|
||||
|
||||
@@ -205,6 +205,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
MLA_LAYER,
|
||||
)
|
||||
|
||||
@@ -259,6 +260,7 @@ def test_skipped_k_cache_insert_accepts_no_k(
|
||||
topk_indices,
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
"",
|
||||
)
|
||||
|
||||
@@ -267,9 +269,11 @@ def test_skipped_k_cache_insert_accepts_no_k(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fp8_query", [False, True])
|
||||
@pytest.mark.parametrize("prepared_mha", [False, True])
|
||||
def test_deepseek_v32_dispatches_selected_mha(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_query: bool,
|
||||
prepared_mha: bool,
|
||||
) -> None:
|
||||
attn_metadata = SimpleNamespace(num_actual_tokens=2)
|
||||
kv_cache = torch.empty(1)
|
||||
@@ -301,7 +305,11 @@ def test_deepseek_v32_dispatches_selected_mha(
|
||||
q_nope = torch.randn(2, 1, 2)
|
||||
q_pe = torch.randn(2, 1, 2)
|
||||
mqa_q = torch.randn(2, 1, 2)
|
||||
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
|
||||
if fp8_query and not prepared_mha:
|
||||
mqa_q = mqa_q.to(torch.float8_e4m3fn)
|
||||
mha_q = torch.cat(
|
||||
(q_nope, q_pe + 1 if fp8_query and not prepared_mha else mqa_q), dim=-1
|
||||
)
|
||||
kv_c = torch.empty(2, 2)
|
||||
k_pe = torch.empty(2, 2)
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
@@ -263,6 +265,23 @@ class TestFusedMMInputNormKernel:
|
||||
class TestFusedMMInputNormConstruction:
|
||||
"""Weight/bias buffer semantics at init time."""
|
||||
|
||||
def test_input_norm_is_always_enabled(self):
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
compile_mm_encoder=True,
|
||||
custom_ops=["none"],
|
||||
)
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
FusedMMInputNorm(_RGB_MEAN, _RGB_STD, _RGB_RESCALE)
|
||||
assert (
|
||||
config.compilation_config.enabled_custom_ops["fused_mm_input_norm"] == 1
|
||||
)
|
||||
assert not config.compilation_config.disabled_custom_ops[
|
||||
"fused_mm_input_norm"
|
||||
]
|
||||
|
||||
def test_identity_config_buffers(self):
|
||||
"""Numerically identity parameters still build plain weight/bias
|
||||
buffers; the identity shortcut lives in ``from_model_config``."""
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import (
|
||||
TokenEmbeddingPoolerHead,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
_HIDDEN = 16
|
||||
@@ -24,7 +25,7 @@ _BATCH = 3
|
||||
def _make_params(
|
||||
n: int,
|
||||
*,
|
||||
task: str = "embed",
|
||||
task: PoolingTask = "embed",
|
||||
dimensions: int | None = None,
|
||||
use_activation: bool | None = None,
|
||||
) -> list[PoolingParams]:
|
||||
@@ -320,6 +321,7 @@ class TestTokenEmbeddingPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_projector(self):
|
||||
@@ -328,6 +330,7 @@ class TestTokenEmbeddingPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 8)
|
||||
assert torch.allclose(out, proj(x))
|
||||
|
||||
@@ -336,6 +339,7 @@ class TestTokenEmbeddingPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
assert torch.equal(out, x[..., :4])
|
||||
|
||||
@@ -360,6 +364,7 @@ class TestTokenEmbeddingPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
assert torch.equal(out, proj(x)[..., :4])
|
||||
|
||||
@@ -368,6 +373,7 @@ class TestTokenEmbeddingPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
|
||||
@@ -416,6 +422,7 @@ class TestTokenClassifierPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_classifier(self):
|
||||
@@ -424,6 +431,7 @@ class TestTokenClassifierPoolerHead:
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 3)
|
||||
assert torch.allclose(out, clf(x))
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.pooler.tokwise.methods import (
|
||||
get_tok_pooling_method,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
|
||||
|
||||
_CPU = torch.device("cpu")
|
||||
@@ -66,7 +67,7 @@ def _make_pooling_cursor(
|
||||
def _make_metadata(
|
||||
prompt_lens: list[int],
|
||||
*,
|
||||
tasks: list[str] | None = None,
|
||||
tasks: list[PoolingTask] | None = None,
|
||||
token_ids: list[list[int]] | None = None,
|
||||
pooling_params: list[PoolingParams] | None = None,
|
||||
num_scheduled_tokens: list[int] | None = None,
|
||||
|
||||
@@ -36,7 +36,7 @@ try:
|
||||
import tensorizer
|
||||
from tensorizer import EncryptionParams
|
||||
except ImportError:
|
||||
tensorizer = PlaceholderModule("tensorizer") # type: ignore[assignment]
|
||||
tensorizer = PlaceholderModule("tensorizer")
|
||||
EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
|
||||
|
||||
|
||||
|
||||
@@ -250,12 +250,12 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
|
||||
|
||||
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
|
||||
assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)]
|
||||
assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)]
|
||||
|
||||
dp_third_node = _parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
|
||||
)
|
||||
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)]
|
||||
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)]
|
||||
|
||||
dp_tp = _parallel(
|
||||
tensor_parallel_size=2,
|
||||
@@ -263,29 +263,43 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
|
||||
data_parallel_size_local=2,
|
||||
data_parallel_rank=2,
|
||||
)
|
||||
assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)]
|
||||
assert plan_local_ranks(dp_tp) == [
|
||||
(0, 2, 0, 0),
|
||||
(1, 2, 0, 1),
|
||||
(2, 3, 0, 0),
|
||||
(3, 3, 0, 1),
|
||||
]
|
||||
|
||||
# DP + nnodes: node-local TP replicas
|
||||
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
|
||||
assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)]
|
||||
assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)]
|
||||
dp_tp_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)]
|
||||
assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)]
|
||||
|
||||
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
|
||||
tp_span_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)]
|
||||
assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)]
|
||||
tp_span_node2 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)]
|
||||
assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)]
|
||||
|
||||
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
|
||||
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)]
|
||||
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)]
|
||||
|
||||
# PP adds one daemon placement per stage while preserving TP rank order.
|
||||
pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2)
|
||||
assert plan_local_ranks(pp_tp) == [
|
||||
(0, 0, 0, 0),
|
||||
(1, 0, 0, 1),
|
||||
(2, 0, 1, 0),
|
||||
(3, 0, 1, 1),
|
||||
]
|
||||
|
||||
|
||||
def test_daemon_rejects_unmappable_parallelism():
|
||||
@@ -302,8 +316,7 @@ def test_daemon_rejects_unmappable_parallelism():
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
)
|
||||
with pytest.raises(ValueError, match="pipeline"):
|
||||
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
|
||||
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
@@ -335,6 +348,24 @@ def test_weight_cache_key_distinguishes_dp_ranks():
|
||||
vllm_version="v",
|
||||
dp_size=16,
|
||||
dp_rank=3,
|
||||
pp_size=2,
|
||||
pp_rank=1,
|
||||
)
|
||||
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
|
||||
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
|
||||
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
|
||||
|
||||
|
||||
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
|
||||
"""Copy mode clones the weights into the engine, so nothing is external
|
||||
(vllm_config is never touched)."""
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.model_executor.model_loader.weight_cache.ipc_loader import (
|
||||
IpcModelLoader,
|
||||
)
|
||||
|
||||
copy_mode = LoadConfig(
|
||||
load_format="ipc_cache", model_loader_extra_config={"mode": "copy"}
|
||||
)
|
||||
loader = IpcModelLoader(copy_mode)
|
||||
assert loader.get_external_weight_memory(None) == 0
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.testing._internal.two_tensor import TwoTensor
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.model_loader.weight_checksum import (
|
||||
compute_tensor_digests,
|
||||
zero_weights,
|
||||
)
|
||||
from vllm.v1.worker import gpu_worker
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _Model(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.linear = nn.Linear(2, 2)
|
||||
nn.init.ones_(self.linear.weight)
|
||||
nn.init.ones_(self.linear.bias)
|
||||
# A strided view, which cannot be reinterpreted as bytes in place.
|
||||
self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2])
|
||||
self.scalar = nn.Parameter(torch.tensor(2.0))
|
||||
# Loading never restores buffers, so reset must leave them alone.
|
||||
self.register_buffer("k_scale", torch.tensor(2.0))
|
||||
|
||||
|
||||
def test_digests_cover_every_parameter():
|
||||
assert set(compute_tensor_digests(_Model())) == {
|
||||
"linear.weight",
|
||||
"linear.bias",
|
||||
"strided",
|
||||
"scalar",
|
||||
}
|
||||
|
||||
|
||||
def test_zero_weights_changes_every_parameter_and_no_buffer():
|
||||
model = _Model()
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
after = compute_tensor_digests(model)
|
||||
assert all(before[name] != after[name] for name in before)
|
||||
assert model.k_scale.item() == 2.0
|
||||
|
||||
|
||||
def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch):
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
model = nn.Module()
|
||||
model.transform = parameter.SharedWeightParameter(weight_loader=None)
|
||||
model.transform.add_partition(0, object(), 2, 2)
|
||||
model.transform.partitions[0].data.fill_(3.0)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"transform.0"}
|
||||
assert compute_tensor_digests(model) != before
|
||||
|
||||
|
||||
def test_tensor_subclass_inner_tensors_are_hashed_and_reset():
|
||||
model = nn.Module()
|
||||
model.weight = nn.Parameter(
|
||||
TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False
|
||||
)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"weight.a", "weight.b"}
|
||||
assert all(
|
||||
before[name] != digest for name, digest in compute_tensor_digests(model).items()
|
||||
)
|
||||
|
||||
|
||||
def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch):
|
||||
for name in ("get_tp_group", "get_pp_group", "get_pcp_group"):
|
||||
monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0))
|
||||
prefixes = []
|
||||
for dp_rank in (0, 1):
|
||||
worker = object.__new__(Worker)
|
||||
worker.parallel_config = ParallelConfig(
|
||||
data_parallel_size=2, data_parallel_rank=dp_rank
|
||||
)
|
||||
# Dense DP resets data_parallel_rank to 0 in each engine.
|
||||
worker.parallel_config.reconfigure_for_independent_dp_rank()
|
||||
prefixes.append(worker._weight_checksum_prefix())
|
||||
assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"]
|
||||
@@ -6,10 +6,61 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers import utils
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
|
||||
def test_dispatch_prepacks_arm_bf16_causal_conv(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
|
||||
)
|
||||
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
|
||||
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
|
||||
|
||||
packed = torch.randn(32, 4, dtype=torch.bfloat16)
|
||||
pack_inputs = []
|
||||
|
||||
def pack(weight):
|
||||
pack_inputs.append(weight.clone())
|
||||
return packed
|
||||
|
||||
monkeypatch.setattr(utils.ops, "causal_conv1d_weight_pack", pack)
|
||||
original = torch.randn(32, 1, 4, dtype=torch.bfloat16)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(original.clone(), requires_grad=False)
|
||||
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
|
||||
assert len(pack_inputs) == 1
|
||||
# Check causal_conv1d_weight_pack received the expected weights
|
||||
torch.testing.assert_close(pack_inputs[0], original.view(32, 4))
|
||||
# Check we have correctly stashed the unpacked weights
|
||||
torch.testing.assert_close(layer._cpu_unpacked_conv_weight, original.view(32, 4))
|
||||
# Check we have correctly stored the packed weights
|
||||
assert layer.weight.data_ptr() == packed.data_ptr()
|
||||
|
||||
|
||||
def test_dispatch_does_not_pack_3d_expert_weight(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
|
||||
)
|
||||
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
|
||||
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
|
||||
pack_calls = []
|
||||
monkeypatch.setattr(
|
||||
utils.ops,
|
||||
"causal_conv1d_weight_pack",
|
||||
lambda weight: pack_calls.append(weight),
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.randn(8, 32, 4, dtype=torch.bfloat16), requires_grad=False
|
||||
)
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
assert pack_calls == []
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _mock_zentorch_linear_unary():
|
||||
"""Register a mock zentorch_linear_unary op when zentorch is not installed.
|
||||
|
||||
@@ -1,16 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from inspect import signature
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.warmup import kernel_warmup as warmup
|
||||
from vllm.model_executor.warmup.kernel_warmup import (
|
||||
_flashinfer_autotune_token_counts,
|
||||
_run_flashinfer_autotune_dummy_runs,
|
||||
flashinfer_autotune,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend)
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend),
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
),
|
||||
get_model=Mock(
|
||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||
@@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class _AutotuneGroup:
|
||||
def __init__(self, run, ranks):
|
||||
self.run = run
|
||||
self.ranks = tuple(ranks)
|
||||
self.world_size = len(self.ranks)
|
||||
self.rank_in_group = self.ranks.index(run.rank)
|
||||
self.cpu_group = self
|
||||
|
||||
def record(self, operation):
|
||||
self.run.collectives[self.ranks][self.run.rank].append(operation)
|
||||
|
||||
def broadcast_object(self, obj, src=0):
|
||||
assert src == 0
|
||||
self.record(("broadcast", src))
|
||||
if self.rank_in_group == src:
|
||||
self.run.broadcasts[self.ranks] = obj
|
||||
return self.run.broadcasts[self.ranks]
|
||||
|
||||
def barrier(self):
|
||||
self.record(("barrier",))
|
||||
|
||||
|
||||
class _AutotuneTuner:
|
||||
def __init__(self, run):
|
||||
self.run = run
|
||||
self.cache = {}
|
||||
self.loaded = None
|
||||
|
||||
def load_configs(self, path):
|
||||
self.loaded = json.loads(Path(path).read_text())
|
||||
self.cache.update(self.loaded)
|
||||
|
||||
def save_configs(self, path):
|
||||
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
|
||||
Path(path).write_text(json.dumps(self.cache))
|
||||
|
||||
def profile(self, operation):
|
||||
if operation in self.cache:
|
||||
return
|
||||
group = self.run.tuning_group
|
||||
self.run.profile_groups[self.run.rank].append(
|
||||
None if group is None else group.ranks
|
||||
)
|
||||
for tactic in range(2):
|
||||
if group is not None:
|
||||
group.record(("all_reduce", operation, tactic))
|
||||
self.cache[operation] = self.run.rank // self.run.tp
|
||||
|
||||
|
||||
class _AutotuneRun:
|
||||
def __init__(self, pp, tp):
|
||||
self.pp, self.tp = pp, tp
|
||||
self.rank = 0
|
||||
self.tuning_group = None
|
||||
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
|
||||
defaultdict(lambda: defaultdict(list))
|
||||
)
|
||||
self.broadcasts = {}
|
||||
self.tuners = {}
|
||||
self.saves = []
|
||||
self.profile_groups = defaultdict(list)
|
||||
|
||||
def world(self):
|
||||
return _AutotuneGroup(self, range(self.pp * self.tp))
|
||||
|
||||
def tensor_group(self):
|
||||
start = self.rank // self.tp * self.tp
|
||||
return _AutotuneGroup(self, range(start, start + self.tp))
|
||||
|
||||
def pipeline_group(self):
|
||||
return SimpleNamespace(world_size=self.pp)
|
||||
|
||||
def set_group(self, group):
|
||||
self.tuning_group = group
|
||||
|
||||
def dummy_runs(self, runner, *, skip_attn=False):
|
||||
self.tuners[self.rank].profile("shared_gemm")
|
||||
if self.pp > 1 and self.rank // self.tp == 0:
|
||||
self.tuners[self.rank].profile("pp0_extra_gemm")
|
||||
|
||||
def execute(self):
|
||||
for rank in range(self.pp * self.tp):
|
||||
self.rank = rank
|
||||
self.tuners[rank] = _AutotuneTuner(self)
|
||||
flashinfer_autotune(_make_runner([]))
|
||||
return self
|
||||
|
||||
def assert_collectives_match(self):
|
||||
for ranks, traces in self.collectives.items():
|
||||
assert set(traces) == set(ranks)
|
||||
expected = traces[ranks[0]]
|
||||
assert all(trace == expected for trace in traces.values()), dict(traces)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def autotune_run(monkeypatch, tmp_path):
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed import parallel_state
|
||||
|
||||
def make_run(*, pp=2, tp=4):
|
||||
run = _AutotuneRun(pp, tp)
|
||||
autotuner = ModuleType("flashinfer.autotuner")
|
||||
monkeypatch.setattr(
|
||||
autotuner,
|
||||
"AutoTuner",
|
||||
SimpleNamespace(get=lambda: run.tuners[run.rank]),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
autotuner, "set_autotune_process_group", run.set_group, raising=False
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
|
||||
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
|
||||
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
|
||||
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
|
||||
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
warmup,
|
||||
"resolve_flashinfer_autotune_file",
|
||||
lambda runner: tmp_path / "autotune_configs.json",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
|
||||
)
|
||||
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
|
||||
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
|
||||
return run
|
||||
|
||||
return make_run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp", [1, 4])
|
||||
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
|
||||
run = autotune_run(tp=tp).execute()
|
||||
run.assert_collectives_match()
|
||||
for rank, groups in run.profile_groups.items():
|
||||
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
|
||||
assert groups and all(
|
||||
group == (expected if tp > 1 else None) for group in groups
|
||||
)
|
||||
|
||||
|
||||
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
||||
autotune_run, tmp_path
|
||||
):
|
||||
legacy = tmp_path / "autotune_configs.json"
|
||||
legacy.write_text('{"legacy_world_cache": 99}')
|
||||
cold = autotune_run().execute()
|
||||
assert [rank for rank, _, _ in cold.saves] == [0, 4]
|
||||
paths = [path for _, path, _ in cold.saves]
|
||||
assert len(set(paths)) == 2 and legacy not in paths
|
||||
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
||||
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
|
||||
assert cold.saves[1][2] == {"shared_gemm": 1}
|
||||
cold.assert_collectives_match()
|
||||
warm = autotune_run().execute()
|
||||
warm.assert_collectives_match()
|
||||
assert not warm.profile_groups
|
||||
for rank, tuner in warm.tuners.items():
|
||||
assert tuner.loaded == cold.saves[rank // 4][2]
|
||||
paths[1].unlink()
|
||||
mixed = autotune_run().execute()
|
||||
mixed.assert_collectives_match()
|
||||
assert set(mixed.profile_groups) == {4, 5, 6, 7}
|
||||
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
|
||||
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
|
||||
|
||||
|
||||
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
|
||||
run = autotune_run(pp=1, tp=4).execute()
|
||||
run.assert_collectives_match()
|
||||
assert [(rank, path.name) for rank, path, _ in run.saves] == [
|
||||
(0, "autotune_configs.json")
|
||||
]
|
||||
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
|
||||
|
||||
@@ -0,0 +1,636 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.model_executor.models.k2_horizon as k2_horizon
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.k2_horizon import (
|
||||
K2HorizonDecoderLayer,
|
||||
K2HorizonModel,
|
||||
K2HorizonRMSNorm,
|
||||
_rope_weight_perm,
|
||||
apply_partial_rope,
|
||||
is_rope_weights_folding_supported,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
DTYPE = torch.bfloat16
|
||||
SHAPES = [
|
||||
(128, 64),
|
||||
(128, 128),
|
||||
(192, 128),
|
||||
(96, 32),
|
||||
]
|
||||
|
||||
|
||||
def get_gptj_rope(
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
max_position: int,
|
||||
base: float,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
rotary_emb = get_rope(
|
||||
rope_head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=True,
|
||||
rope_parameters={"rope_theta": base},
|
||||
dtype=dtype,
|
||||
)
|
||||
return apply_partial_rope(
|
||||
rotary_emb,
|
||||
positions,
|
||||
q,
|
||||
k,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
fold_rope_weights=False,
|
||||
)
|
||||
|
||||
|
||||
def _apply_head_perm(
|
||||
x: torch.Tensor, num: int, head_dim: int, idx: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return (
|
||||
x.reshape(*x.shape[:-1], num, head_dim)[..., idx].reshape(*x.shape).contiguous()
|
||||
)
|
||||
|
||||
|
||||
def get_neox_rope(
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
max_position: int,
|
||||
base: float,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
rotary_emb = get_rope(
|
||||
head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=True,
|
||||
rope_parameters={"rope_theta": base, "rope_dim": rope_head_dim},
|
||||
dtype=dtype,
|
||||
)
|
||||
return apply_partial_rope(
|
||||
rotary_emb,
|
||||
positions,
|
||||
q,
|
||||
k,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
fold_rope_weights=True,
|
||||
)
|
||||
|
||||
|
||||
def fold_qk_proj_weight(
|
||||
weight: torch.Tensor, head_dim: int, idx: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
hidden = weight.shape[-1]
|
||||
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
|
||||
|
||||
|
||||
def fold_qk_channels(
|
||||
vec: torch.Tensor, head_dim: int, idx: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
def test_rope_fold_matches_permute_then_rope(
|
||||
head_dim, rope_head_dim, default_vllm_config
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_tokens = 17
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
hidden_size = 512
|
||||
max_position = 4096
|
||||
base = 10000.0
|
||||
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
positions = torch.randint(0, max_position, (num_tokens,), device=DEVICE)
|
||||
|
||||
scale = hidden_size**-0.5
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
|
||||
w_q = (
|
||||
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
|
||||
* scale
|
||||
)
|
||||
w_k = (
|
||||
torch.randn(num_kv_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
|
||||
* scale
|
||||
)
|
||||
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
|
||||
|
||||
q = torch.nn.functional.linear(x, w_q)
|
||||
k = torch.nn.functional.linear(x, w_k)
|
||||
q_gptj, k_gptj = get_gptj_rope(
|
||||
positions,
|
||||
q,
|
||||
k,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
max_position,
|
||||
base,
|
||||
dtype,
|
||||
)
|
||||
|
||||
q_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_q, head_dim, idx))
|
||||
k_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_k, head_dim, idx))
|
||||
q_neox, k_neox = get_neox_rope(
|
||||
positions,
|
||||
q_folded,
|
||||
k_folded,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
max_position,
|
||||
base,
|
||||
dtype,
|
||||
)
|
||||
|
||||
q_gptj_perm = _apply_head_perm(q_gptj, num_heads, head_dim, idx)
|
||||
k_gptj_perm = _apply_head_perm(k_gptj, num_kv_heads, head_dim, idx)
|
||||
|
||||
atol, rtol = 1e-2, 1e-2
|
||||
|
||||
# Two complementary checks. First, an element-wise check that the folded
|
||||
# NeoX path equals the GPT-J path up to the channel permutation P
|
||||
# (neox_rope(Pq, Pk) == P @ gptj_rope(q, k)), hence permuting q_gptj/k_gptj.
|
||||
# Second, the scores check below: P cancels in q @ k.T.
|
||||
torch.testing.assert_close(q_neox, q_gptj_perm, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(k_neox, k_gptj_perm, atol=atol, rtol=rtol)
|
||||
|
||||
def scores(qt, kt):
|
||||
qh = qt.reshape(num_tokens, num_heads, head_dim)
|
||||
kh = kt.reshape(num_tokens, num_kv_heads, head_dim)
|
||||
g = num_heads // num_kv_heads
|
||||
qh = qh.reshape(num_tokens, num_kv_heads, g, head_dim).float()
|
||||
kh = kh.float()
|
||||
return torch.einsum("tkgd,skd->tksg", qh, kh)
|
||||
|
||||
torch.testing.assert_close(
|
||||
scores(q_neox, k_neox), scores(q_gptj, k_gptj), atol=atol, rtol=rtol
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("num_heads", [8, 2])
|
||||
def test_qk_bias_fold_matches_permute(
|
||||
head_dim, rope_head_dim, num_heads, default_vllm_config
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_tokens = 17
|
||||
hidden_size = 512
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
scale = hidden_size**-0.5
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
|
||||
w = (
|
||||
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
|
||||
* scale
|
||||
)
|
||||
b = torch.randn(num_heads * head_dim, dtype=dtype, device=DEVICE)
|
||||
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
|
||||
|
||||
ref = _apply_head_perm(
|
||||
torch.nn.functional.linear(x, w, b), num_heads, head_dim, idx
|
||||
)
|
||||
folded = torch.nn.functional.linear(
|
||||
x,
|
||||
fold_qk_proj_weight(w, head_dim, idx),
|
||||
fold_qk_channels(b, head_dim, idx),
|
||||
)
|
||||
|
||||
atol, rtol = 1e-2, 1e-2
|
||||
torch.testing.assert_close(folded, ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a GPU (CUDA/ROCm)"
|
||||
)
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("num_heads", [8, 2])
|
||||
def test_qk_norm_scale_fold_matches_permute(
|
||||
head_dim, rope_head_dim, num_heads, default_vllm_config
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_tokens = 13
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
hidden = num_heads * head_dim
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
|
||||
|
||||
x = torch.randn(num_tokens, hidden, dtype=dtype, device=DEVICE)
|
||||
|
||||
norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
|
||||
with torch.no_grad():
|
||||
norm.weight.copy_(torch.randn(hidden, dtype=dtype, device=DEVICE))
|
||||
|
||||
y_ref = _apply_head_perm(norm(x.clone()), num_heads, head_dim, idx)
|
||||
|
||||
folded_norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
|
||||
with torch.no_grad():
|
||||
folded_norm.weight.copy_(fold_qk_channels(norm.weight, head_dim, idx))
|
||||
# Permute x to emulate the folded q_proj/k_proj: at runtime the norm sees the
|
||||
# already-permuted projection output (P @ x), not the raw activations.
|
||||
y_folded = folded_norm(_apply_head_perm(x.clone(), num_heads, head_dim, idx))
|
||||
|
||||
atol, rtol = 1e-2, 1e-2
|
||||
torch.testing.assert_close(y_folded, y_ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
class _Fp8QKVOnlyConfig(Fp8Config):
|
||||
def get_quant_method(self, layer, prefix):
|
||||
if isinstance(layer, LinearBase) and not (
|
||||
prefix.endswith("qkv_proj") or prefix.endswith("qk_proj")
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
return super().get_quant_method(layer, prefix)
|
||||
|
||||
|
||||
class _DummyAttention(nn.Module):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__()
|
||||
|
||||
def forward(self, q, k, v):
|
||||
return q
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dist_init():
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
|
||||
local_rank=0,
|
||||
backend="gloo",
|
||||
)
|
||||
initialize_model_parallel(1, 1)
|
||||
yield
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _make_hf_config(
|
||||
*,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
attention_bias: bool,
|
||||
query_key_norm: bool,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hidden_size=hidden_size,
|
||||
max_position_embeddings=4096,
|
||||
dual_chunk_attention_config=None,
|
||||
mlp_only_layers=[],
|
||||
num_experts=0, # dense layer: avoids the MoE path
|
||||
decoder_sparse_step=1,
|
||||
mova_num_experts=0,
|
||||
num_attention_heads=num_heads,
|
||||
num_key_value_heads=num_kv_heads,
|
||||
rope_parameters={"rope_theta": 10000.0},
|
||||
rope_head_dim=rope_head_dim,
|
||||
query_key_norm=query_key_norm,
|
||||
rms_norm_eps=1e-6,
|
||||
attention_bias=attention_bias,
|
||||
head_dim=head_dim,
|
||||
attention_gate_func=None,
|
||||
intermediate_size=2 * hidden_size,
|
||||
hidden_act="silu",
|
||||
layernorm_num_groups=1,
|
||||
vocab_size=1000,
|
||||
num_hidden_layers=1,
|
||||
tie_word_embeddings=False,
|
||||
)
|
||||
|
||||
|
||||
def _build_single_layer_model(
|
||||
monkeypatch, hf_config: SimpleNamespace, *, quant_config=None
|
||||
) -> K2HorizonModel:
|
||||
monkeypatch.setattr(k2_horizon, "Attention", _DummyAttention)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
hf_text_config=hf_config, dtype=torch.get_default_dtype()
|
||||
)
|
||||
vllm_config.quant_config = quant_config
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer = K2HorizonDecoderLayer(vllm_config=vllm_config, prefix="model.layers.0")
|
||||
|
||||
model = K2HorizonModel.__new__(K2HorizonModel)
|
||||
nn.Module.__init__(model)
|
||||
model.config = hf_config
|
||||
model.quant_config = quant_config
|
||||
model.num_redundant_experts = 0
|
||||
model.layers = nn.ModuleList([layer])
|
||||
return model
|
||||
|
||||
|
||||
def _fold_rows(weight: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
|
||||
hidden = weight.shape[-1]
|
||||
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
|
||||
|
||||
|
||||
def _fold_channels(vec: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
|
||||
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
|
||||
|
||||
|
||||
def _non_qk_weights(prefix, *, hidden_size, intermediate_size, q_size, device=None):
|
||||
return [
|
||||
(
|
||||
f"{prefix}.self_attn.o_proj.weight",
|
||||
torch.randn(hidden_size, q_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.mlp.gate_proj.weight",
|
||||
torch.randn(intermediate_size, hidden_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.mlp.up_proj.weight",
|
||||
torch.randn(intermediate_size, hidden_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.mlp.down_proj.weight",
|
||||
torch.randn(hidden_size, intermediate_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.input_layernorm.weight",
|
||||
torch.randn(hidden_size, device=device),
|
||||
),
|
||||
(
|
||||
f"{prefix}.post_attention_layernorm.weight",
|
||||
torch.randn(hidden_size, device=device),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("attention_bias", [False, True])
|
||||
def test_load_weights_dense_folds_qk(
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
attention_bias,
|
||||
monkeypatch,
|
||||
):
|
||||
dtype = DTYPE
|
||||
set_random_seed(0)
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
hidden_size = 128
|
||||
intermediate_size = 2 * hidden_size
|
||||
q_size = num_heads * head_dim
|
||||
kv_size = num_kv_heads * head_dim
|
||||
prefix = "layers.0"
|
||||
|
||||
with set_default_torch_dtype(dtype):
|
||||
hf_config = _make_hf_config(
|
||||
hidden_size=hidden_size,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
attention_bias=attention_bias,
|
||||
query_key_norm=True,
|
||||
)
|
||||
model = _build_single_layer_model(monkeypatch, hf_config, quant_config=None)
|
||||
|
||||
assert model.layers[0].self_attn.fold_rope_weights is True
|
||||
|
||||
raw = {
|
||||
"q_proj.weight": torch.randn(q_size, hidden_size),
|
||||
"k_proj.weight": torch.randn(kv_size, hidden_size),
|
||||
"v_proj.weight": torch.randn(kv_size, hidden_size),
|
||||
"q_norm.weight": torch.randn(q_size),
|
||||
"k_norm.weight": torch.randn(kv_size),
|
||||
}
|
||||
if attention_bias:
|
||||
raw.update(
|
||||
{
|
||||
"q_proj.bias": torch.randn(q_size),
|
||||
"k_proj.bias": torch.randn(kv_size),
|
||||
"v_proj.bias": torch.randn(kv_size),
|
||||
"o_proj.bias": torch.randn(hidden_size),
|
||||
}
|
||||
)
|
||||
|
||||
weights: list[tuple[str, torch.Tensor]] = [
|
||||
(f"{prefix}.self_attn.{name}", tensor) for name, tensor in raw.items()
|
||||
]
|
||||
weights += _non_qk_weights(
|
||||
prefix,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
q_size=q_size,
|
||||
)
|
||||
|
||||
loaded = model.load_weights(iter(weights))
|
||||
|
||||
params = dict(model.named_parameters())
|
||||
assert loaded == set(params)
|
||||
|
||||
idx = _rope_weight_perm(head_dim, rope_head_dim)
|
||||
|
||||
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
|
||||
q_part = qkv[:q_size]
|
||||
k_part = qkv[q_size : q_size + kv_size]
|
||||
v_part = qkv[q_size + kv_size :]
|
||||
|
||||
q_norm = params[f"{prefix}.self_attn.q_norm.weight"]
|
||||
k_norm = params[f"{prefix}.self_attn.k_norm.weight"]
|
||||
|
||||
assert torch.equal(q_part, _fold_rows(raw["q_proj.weight"], head_dim, idx))
|
||||
assert torch.equal(k_part, _fold_rows(raw["k_proj.weight"], head_dim, idx))
|
||||
assert torch.equal(v_part, raw["v_proj.weight"]) # v never folded
|
||||
assert torch.equal(q_norm, _fold_channels(raw["q_norm.weight"], head_dim, idx))
|
||||
assert torch.equal(k_norm, _fold_channels(raw["k_norm.weight"], head_dim, idx))
|
||||
assert qkv.dtype == dtype
|
||||
|
||||
if attention_bias:
|
||||
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
|
||||
q_bias = qkv_bias[:q_size]
|
||||
k_bias = qkv_bias[q_size : q_size + kv_size]
|
||||
v_bias = qkv_bias[q_size + kv_size :]
|
||||
assert torch.equal(q_bias, _fold_channels(raw["q_proj.bias"], head_dim, idx))
|
||||
assert torch.equal(k_bias, _fold_channels(raw["k_proj.bias"], head_dim, idx))
|
||||
assert torch.equal(v_bias, raw["v_proj.bias"]) # v bias never folded
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported.",
|
||||
)
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
|
||||
@pytest.mark.parametrize("attention_bias", [False, True])
|
||||
def test_load_weights_fp8_qk_no_fold(
|
||||
head_dim,
|
||||
rope_head_dim,
|
||||
attention_bias,
|
||||
monkeypatch,
|
||||
):
|
||||
set_random_seed(0)
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
hidden_size = 128
|
||||
intermediate_size = 2 * hidden_size
|
||||
q_size = num_heads * head_dim
|
||||
kv_size = num_kv_heads * head_dim
|
||||
prefix = "layers.0"
|
||||
|
||||
def rand_fp8(*shape):
|
||||
return (torch.randn(*shape, device=DEVICE) * 32.0).to(torch.float8_e4m3fn)
|
||||
|
||||
def rand_scale():
|
||||
return torch.rand(1, device=DEVICE, dtype=torch.float32) + 0.1
|
||||
|
||||
with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE):
|
||||
hf_config = _make_hf_config(
|
||||
hidden_size=hidden_size,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
attention_bias=attention_bias,
|
||||
query_key_norm=True,
|
||||
)
|
||||
quant_config = _Fp8QKVOnlyConfig(
|
||||
is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"
|
||||
)
|
||||
model = _build_single_layer_model(
|
||||
monkeypatch, hf_config, quant_config=quant_config
|
||||
)
|
||||
|
||||
qkv_proj = model.layers[0].self_attn.qkv_proj
|
||||
assert isinstance(qkv_proj.quant_method, Fp8LinearMethod)
|
||||
assert model.layers[0].self_attn.fold_rope_weights is False
|
||||
|
||||
raw_w = {
|
||||
"q_proj.weight": rand_fp8(q_size, hidden_size),
|
||||
"k_proj.weight": rand_fp8(kv_size, hidden_size),
|
||||
"v_proj.weight": rand_fp8(kv_size, hidden_size),
|
||||
}
|
||||
raw_s = {
|
||||
"q_proj.weight_scale": rand_scale(),
|
||||
"k_proj.weight_scale": rand_scale(),
|
||||
"v_proj.weight_scale": rand_scale(),
|
||||
}
|
||||
q_norm = torch.randn(q_size, device=DEVICE)
|
||||
k_norm = torch.randn(kv_size, device=DEVICE)
|
||||
|
||||
weights: list[tuple[str, torch.Tensor]] = [
|
||||
(f"{prefix}.self_attn.{name}", tensor)
|
||||
for name, tensor in {**raw_w, **raw_s}.items()
|
||||
]
|
||||
weights += [
|
||||
(f"{prefix}.self_attn.q_norm.weight", q_norm),
|
||||
(f"{prefix}.self_attn.k_norm.weight", k_norm),
|
||||
]
|
||||
biases = {}
|
||||
if attention_bias:
|
||||
biases = {
|
||||
"q_proj.bias": torch.randn(q_size, device=DEVICE),
|
||||
"k_proj.bias": torch.randn(kv_size, device=DEVICE),
|
||||
"v_proj.bias": torch.randn(kv_size, device=DEVICE),
|
||||
"o_proj.bias": torch.randn(hidden_size, device=DEVICE),
|
||||
}
|
||||
weights += [
|
||||
(f"{prefix}.self_attn.{name}", tensor)
|
||||
for name, tensor in biases.items()
|
||||
]
|
||||
weights += _non_qk_weights(
|
||||
prefix,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
q_size=q_size,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
loaded = model.load_weights(iter(weights))
|
||||
|
||||
params = dict(model.named_parameters())
|
||||
assert loaded == set(params)
|
||||
|
||||
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
|
||||
weight_scale = params[f"{prefix}.self_attn.qkv_proj.weight_scale"]
|
||||
|
||||
assert qkv.dtype == torch.float8_e4m3fn
|
||||
assert torch.equal(qkv[:q_size], raw_w["q_proj.weight"])
|
||||
assert torch.equal(qkv[q_size : q_size + kv_size], raw_w["k_proj.weight"])
|
||||
assert torch.equal(qkv[q_size + kv_size :], raw_w["v_proj.weight"])
|
||||
|
||||
assert torch.equal(
|
||||
weight_scale,
|
||||
torch.cat(
|
||||
[
|
||||
raw_s["q_proj.weight_scale"],
|
||||
raw_s["k_proj.weight_scale"],
|
||||
raw_s["v_proj.weight_scale"],
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
assert torch.equal(params[f"{prefix}.self_attn.q_norm.weight"], q_norm)
|
||||
assert torch.equal(params[f"{prefix}.self_attn.k_norm.weight"], k_norm)
|
||||
|
||||
if attention_bias:
|
||||
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
|
||||
assert torch.equal(qkv_bias[:q_size], biases["q_proj.bias"])
|
||||
assert torch.equal(qkv_bias[q_size : q_size + kv_size], biases["k_proj.bias"])
|
||||
assert torch.equal(qkv_bias[q_size + kv_size :], biases["v_proj.bias"])
|
||||
|
||||
|
||||
def test_is_rope_weights_folding_supported(default_vllm_config):
|
||||
dense = SimpleNamespace(quant_method=UnquantizedLinearMethod())
|
||||
quantized = SimpleNamespace(quant_method=object())
|
||||
|
||||
assert is_rope_weights_folding_supported(dense, None) is True
|
||||
assert is_rope_weights_folding_supported(quantized, None) is False
|
||||
assert is_rope_weights_folding_supported(dense, {"chunk_size": 8}) is False
|
||||
assert is_rope_weights_folding_supported(SimpleNamespace(), None) is False
|
||||
@@ -67,8 +67,13 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
with patch.object(
|
||||
warmup, "flashinfer_replayssm_autotune_supported", return_value=True
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
max_num_reqs, decode_kwargs = result
|
||||
assert max_num_reqs == expected_num_reqs
|
||||
expected_kwargs = {
|
||||
**PREFILL_KWARGS,
|
||||
"num_tokens": expected_num_reqs * query_len,
|
||||
@@ -82,7 +87,7 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
force_attention=True,
|
||||
profile_seq_lens=query_len + 1,
|
||||
)
|
||||
assert result == (expected_num_reqs, expected_kwargs)
|
||||
assert decode_kwargs == expected_kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -91,8 +96,14 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
(dict(use_replayssm=False), True),
|
||||
(dict(backend=MambaBackendEnum.TRITON), True),
|
||||
({}, False),
|
||||
(dict(num_blocks=1), True),
|
||||
],
|
||||
ids=[
|
||||
"replayssm_disabled",
|
||||
"non_flashinfer_backend",
|
||||
"kernel_unavailable",
|
||||
"zero_non_padding_slots",
|
||||
],
|
||||
ids=["replayssm_disabled", "non_flashinfer_backend", "kernel_unavailable"],
|
||||
)
|
||||
def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
|
||||
with patch.object(
|
||||
@@ -100,7 +111,9 @@ def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
|
||||
"flashinfer_replayssm_autotune_supported",
|
||||
return_value=flashinfer_supported,
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
@@ -116,10 +129,12 @@ def test_replayssm_autotune_slots_restore_state_and_trackers():
|
||||
)
|
||||
mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32)
|
||||
tracked = (
|
||||
*mixer.kv_cache,
|
||||
mixer._replayssm_ring_start,
|
||||
mixer._replayssm_prev_num_accepted,
|
||||
mixer._replayssm_prev_query_len,
|
||||
)
|
||||
|
||||
block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1)
|
||||
|
||||
Reference in New Issue
Block a user