[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
62f3bf58a5
commit
3116c5d06b
@@ -46,6 +46,39 @@ def test_optional_type():
|
||||
assert optional_type_func("42") == 42
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
["--engram-config", '{"cpu_offload": false, "embedding_across_dp": true}'],
|
||||
[
|
||||
"--engram-config.cpu_offload",
|
||||
"false",
|
||||
"--engram-config.embedding_across_dp",
|
||||
"true",
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_engram_config_cli(options, monkeypatch):
|
||||
"""CLI settings take precedence over the legacy offload environment."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = EngineArgs.from_cli_args(parser.parse_args(options))
|
||||
assert args.engram_config is not None
|
||||
assert args.engram_config.cpu_offload is False
|
||||
assert args.engram_config.embedding_across_dp is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options, provided",
|
||||
[([], False), (["--engram-config", "{}"], True)],
|
||||
)
|
||||
def test_engram_config_cli_optional(options, provided):
|
||||
"""An explicit empty config must remain distinct from an omitted config."""
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = EngineArgs.from_cli_args(parser.parse_args(options))
|
||||
assert (args.engram_config is not None) == provided
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("type_hint", "type", "expected"),
|
||||
[
|
||||
|
||||
@@ -13,27 +13,50 @@ from torch.nn import functional as F
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import (
|
||||
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLELayer,
|
||||
_get_ple_embedding_quant_method,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def _mock_etp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int = 1,
|
||||
all_reduce=lambda tensor: tensor,
|
||||
) -> None:
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=0,
|
||||
world_size=world_size,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
monkeypatch.setattr(ngram_embedding_module, "get_etp_group", lambda: group)
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=world_size),
|
||||
)
|
||||
|
||||
|
||||
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
@@ -171,11 +194,57 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
|
||||
def test_etp_lookup_gathers_and_returns_dp_local_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dp_rank: int,
|
||||
local_tokens: int,
|
||||
) -> None:
|
||||
embedding = Qwen4ExpPLEDeviceEmbedding.__new__(Qwen4ExpPLEDeviceEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.tp_size = 2
|
||||
embedding.etp_data_parallel_size = 2
|
||||
embedding.data_parallel_rank = dp_rank
|
||||
gathered_ids = torch.tensor([[10], [11], [0], [20], [21], [22]])
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=dp_rank,
|
||||
all_gather=lambda input_ids, dim: gathered_ids,
|
||||
all_reduce=lambda embeddings: embeddings,
|
||||
)
|
||||
embedding.parallel_group = group
|
||||
forward_context = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=torch.tensor([2, 3]),
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(ngram_embedding_module, "get_dp_group", lambda: group)
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module, "get_forward_context", lambda: forward_context
|
||||
)
|
||||
local_ids = gathered_ids[dp_rank * 3 : dp_rank * 3 + local_tokens]
|
||||
embeddings = torch.arange(12).reshape(6, 2)
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
assert torch.equal(input_ids, gathered_ids)
|
||||
return embeddings
|
||||
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module.PLEVocabParallelEmbedding,
|
||||
"forward",
|
||||
forward,
|
||||
)
|
||||
output = embedding(local_ids)
|
||||
|
||||
expected = embeddings[dp_rank * 3 : dp_rank * 3 + local_tokens]
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def _make_fp8_embedding_layer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
load_scale: bool = True,
|
||||
) -> embedding_module.VocabParallelEmbedding:
|
||||
) -> Qwen4ExpPLEDeviceEmbedding:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
@@ -185,12 +254,13 @@ def _make_fp8_embedding_layer(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
method = Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
layer = Qwen4ExpPLEDeviceEmbedding(
|
||||
3,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=method,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=method,
|
||||
)
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
|
||||
@@ -228,7 +298,58 @@ def test_ple_fp8_embedding_rejects_missing_global_scale(monkeypatch) -> None:
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
@pytest.mark.parametrize("requires_device_loading", [False, True])
|
||||
@pytest.mark.parametrize("load_scale", [False, True])
|
||||
def test_pinned_ple_post_load_validates_scale_without_staging(
|
||||
monkeypatch: pytest.MonkeyPatch, requires_device_loading: bool, load_scale: bool
|
||||
) -> None:
|
||||
"""Validate scales through the loader while honoring the staging policy."""
|
||||
from contextlib import nullcontext
|
||||
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader import utils as loader_utils
|
||||
|
||||
source = _make_fp8_embedding_layer(monkeypatch, load_scale=load_scale)
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.weight = source.weight
|
||||
embedding.weight_scale = source.weight_scale
|
||||
embedding.embedding_method = source.embedding_method
|
||||
embedding.quant_method = source.embedding_method
|
||||
embedding.tp_rank = source.tp_rank
|
||||
embedding.tp_size = source.tp_size
|
||||
assert QuantizeMethodBase.requires_device_loading
|
||||
assert not embedding.quant_method.requires_device_loading
|
||||
if requires_device_loading:
|
||||
embedding.quant_method.requires_device_loading = True
|
||||
model = nn.Module()
|
||||
model.embedding = embedding
|
||||
weight_ptr = embedding.weight.data_ptr()
|
||||
staged_modules = []
|
||||
|
||||
def track_staging(module, target_device):
|
||||
staged_modules.append(module)
|
||||
return nullcontext()
|
||||
|
||||
monkeypatch.setattr(loader_utils, "device_loading_context", track_staging)
|
||||
monkeypatch.setattr(loader_utils, "maybe_retie_word_embeddings", lambda *args: None)
|
||||
monkeypatch.setattr(
|
||||
loader_utils, "release_device_memory_under_pressure", lambda *args: None
|
||||
)
|
||||
config = SimpleNamespace(quantization="fp8")
|
||||
if load_scale:
|
||||
loader_utils.process_weights_after_loading(model, config, torch.device("cuda"))
|
||||
else:
|
||||
with pytest.raises(ValueError, match="missing its global scale"):
|
||||
loader_utils.process_weights_after_loading(
|
||||
model, config, torch.device("cuda")
|
||||
)
|
||||
assert embedding.weight.device.type == "cpu"
|
||||
assert embedding.weight.data_ptr() == weight_ptr
|
||||
assert staged_modules == ([embedding] if requires_device_loading else [])
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
@@ -251,17 +372,14 @@ def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
all_reduce,
|
||||
)
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
_mock_etp_group(monkeypatch, world_size=2, all_reduce=all_reduce)
|
||||
layer = Qwen4ExpPLEDeviceEmbedding(
|
||||
4,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
)
|
||||
layer.weight.data.copy_(
|
||||
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
|
||||
@@ -283,12 +401,200 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.ignored_layers = [f"{prefix}.shard_0"]
|
||||
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(None, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
|
||||
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
|
||||
|
||||
dynamic_fp8_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=False,
|
||||
ignored_layers=[],
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="serialized FP8"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix)
|
||||
|
||||
|
||||
def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
prefix,
|
||||
"float8_e4m3fn",
|
||||
),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_finalizes_prefetched_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
|
||||
|
||||
def finalize_prefetched(
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert prefetch_output is embedding._prefetch_buffer
|
||||
output.copy_(expected)
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding,
|
||||
"_finalize_prefetch",
|
||||
finalize_prefetched,
|
||||
)
|
||||
|
||||
output = embedding(hidden_states)
|
||||
|
||||
assert output.dtype == embedding._prefetch_buffer.dtype
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.tp_size = 2
|
||||
reduced_dtypes = []
|
||||
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
|
||||
embeddings = torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn)
|
||||
|
||||
output = embedding._reduce_etp_embeddings(embeddings)
|
||||
|
||||
assert reduced_dtypes == [torch.int8]
|
||||
assert output.dtype == embeddings.dtype
|
||||
torch.testing.assert_close(output.float(), embeddings.float())
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_ple_device_embedding_allocates_on_active_device(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
embedding_method = Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding = Qwen4ExpPLEDeviceEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=embedding_method,
|
||||
)
|
||||
|
||||
assert embedding.weight.device == torch.device("cuda:0")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True])
|
||||
def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_checkpoint: bool,
|
||||
) -> None:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding_method = (
|
||||
Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if fp8_checkpoint
|
||||
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
)
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=embedding_method,
|
||||
)
|
||||
|
||||
storage_dtype = torch.float8_e4m3fn if fp8_checkpoint else torch.bfloat16
|
||||
loaded_weight = (
|
||||
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
|
||||
)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = embedding._lookup(input_ids)
|
||||
if fp8_checkpoint:
|
||||
embedding.weight_scale.data.fill_(0.25)
|
||||
weight_ptr = embedding.weight.data_ptr()
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
assert embedding.weight.data_ptr() == weight_ptr
|
||||
dequantized = embedding.dequantize(output, torch.bfloat16)
|
||||
|
||||
assert copied == 4
|
||||
assert embedding.weight.device.type == "cpu"
|
||||
assert embedding.weight.is_pinned()
|
||||
assert output.dtype == storage_dtype
|
||||
expected = loaded_weight[input_ids.cpu()].to(device="cuda:0")
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
expected_scale = 0.25 if fp8_checkpoint else 1.0
|
||||
torch.testing.assert_close(
|
||||
dequantized,
|
||||
expected.to(torch.bfloat16) * expected_scale,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
@@ -310,15 +616,15 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
assert (
|
||||
_get_ple_embedding_quant_method(
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.2.moe.gate_proj",
|
||||
)
|
||||
is None
|
||||
),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
DeviceConfig,
|
||||
EngramConfig,
|
||||
KernelConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
@@ -913,6 +914,159 @@ def test_all2all_backend_has_portable_default():
|
||||
assert ParallelConfig().all2all_backend == "allgather_reducescatter"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dp_size, across_dp, expected",
|
||||
[(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)],
|
||||
)
|
||||
def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: int):
|
||||
parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
data_parallel_size=dp_size,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
config = EngramConfig(embedding_across_dp=across_dp)
|
||||
assert config.get_parallel_size(parallel) == expected
|
||||
|
||||
|
||||
def test_engram_rejects_elastic_cross_dp():
|
||||
parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
data_parallel_size=2,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
parallel.enable_elastic_ep = True
|
||||
with pytest.raises(ValueError, match="embedding_across_dp.*elastic EP"):
|
||||
EngramConfig(embedding_across_dp=True).verify_parallel_config(parallel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", [None, "0", "1"])
|
||||
def test_engram_cpu_offload_environment_fallback(monkeypatch, legacy):
|
||||
"""Explicit settings must override the legacy environment fallback."""
|
||||
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
|
||||
if legacy is not None:
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
assert EngramConfig().cpu_offload == (legacy == "1")
|
||||
assert EngramConfig(cpu_offload=False).cpu_offload is False
|
||||
assert EngramConfig(cpu_offload=True).cpu_offload is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"architecture, ple_layers, cuda, supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", [1], True, True),
|
||||
("Qwen4ExpForConditionalGeneration", [1], True, True),
|
||||
("Qwen4ExpForCausalLM", [], True, False),
|
||||
("Qwen4ExpForCausalLM", None, True, False),
|
||||
("Qwen4ExpForCausalLM", [1], False, False),
|
||||
("LlamaForCausalLM", [1], True, False),
|
||||
("Qwen4ExpMTP", [], True, False),
|
||||
(None, None, True, False),
|
||||
],
|
||||
)
|
||||
def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, supported):
|
||||
"""A similarly named HF field must not enable unsupported implementations."""
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: cuda)
|
||||
model = (
|
||||
cast(
|
||||
ModelConfig,
|
||||
SimpleNamespace(
|
||||
architecture=architecture,
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=ple_layers),
|
||||
),
|
||||
)
|
||||
if architecture is not None
|
||||
else None
|
||||
)
|
||||
config = EngramConfig(cpu_offload=False, embedding_across_dp=False)
|
||||
if supported:
|
||||
config.verify_model_config(model)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
config.verify_model_config(model)
|
||||
|
||||
|
||||
def test_engram_config_defaults_to_none(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
|
||||
config = VllmConfig()
|
||||
assert config.engram_config is None
|
||||
assert config.compute_hash()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", ["0", "1"])
|
||||
def test_engram_none_resolves_legacy_offload(monkeypatch, legacy):
|
||||
"""Legacy enablement materializes a config before model validation."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
config = cast(
|
||||
VllmConfig,
|
||||
SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
architecture="Qwen4ExpForCausalLM",
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=[1]),
|
||||
),
|
||||
speculative_config=None,
|
||||
engram_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
),
|
||||
)
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
if legacy == "1":
|
||||
assert config.engram_config is not None
|
||||
assert config.engram_config.cpu_offload is True
|
||||
else:
|
||||
assert config.engram_config is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", ["0", "1"])
|
||||
def test_engram_explicit_config_requires_supported_model(monkeypatch, legacy):
|
||||
"""Explicit all-false settings still opt into model validation."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig(engram_config=EngramConfig(cpu_offload=False))
|
||||
|
||||
|
||||
def test_engram_legacy_offload_requires_supported_model(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target_has_ple", [False, True])
|
||||
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
|
||||
"""MTP may inherit cross-DP sharding without having its own PLE layers."""
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
target = SimpleNamespace(
|
||||
architecture="Qwen4ExpForCausalLM",
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=[1] if target_has_ple else []),
|
||||
)
|
||||
draft = SimpleNamespace(architecture="Qwen4ExpMTP")
|
||||
config = cast(
|
||||
VllmConfig,
|
||||
SimpleNamespace(
|
||||
model_config=draft,
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=draft, target_model_config=target
|
||||
),
|
||||
engram_config=EngramConfig(embedding_across_dp=True),
|
||||
parallel_config=ParallelConfig(),
|
||||
),
|
||||
)
|
||||
if target_has_ple:
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
|
||||
|
||||
def test_engram_hash_tracks_storage_and_sharding():
|
||||
configs = [
|
||||
EngramConfig(cpu_offload=offload, embedding_across_dp=across_dp)
|
||||
for offload in (False, True)
|
||||
for across_dp in (False, True)
|
||||
]
|
||||
assert len({config.compute_hash() for config in configs}) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("port", [1, 29550, 65535])
|
||||
def test_data_parallel_rpc_port_accepts_valid_ports(port: int):
|
||||
assert ParallelConfig(data_parallel_rpc_port=port).data_parallel_rpc_port == port
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.config.device import DeviceConfig
|
||||
from vllm.config.diffusion import DiffusionConfig
|
||||
from vllm.config.ec_manager_config import EncoderCacheManagerConfig
|
||||
from vllm.config.ec_transfer import ECTransferConfig
|
||||
from vllm.config.engram import EngramConfig
|
||||
from vllm.config.fault_tolerance import FaultToleranceConfig
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
@@ -81,6 +82,8 @@ __all__ = [
|
||||
"EncoderCacheManagerConfig",
|
||||
# From vllm.config.ec_transfer
|
||||
"ECTransferConfig",
|
||||
# From vllm.config.engram
|
||||
"EngramConfig",
|
||||
# From vllm.config.kernel
|
||||
"KernelConfig",
|
||||
# From vllm.config.kv_events
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import config, get_hash_factors, hash_factors
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _default_cpu_offload() -> bool:
|
||||
"""Honor the legacy environment variable only when the field is omitted."""
|
||||
if envs.is_set("VLLM_PLE_CPU_OFFLOAD"):
|
||||
logger.warning_once(
|
||||
"VLLM_PLE_CPU_OFFLOAD is a legacy setting and may be removed in a "
|
||||
"future release. Use --engram-config.cpu_offload instead."
|
||||
)
|
||||
return envs.VLLM_PLE_CPU_OFFLOAD
|
||||
|
||||
|
||||
@config
|
||||
class EngramConfig:
|
||||
"""Configuration for Engram embedding storage and sharding."""
|
||||
|
||||
cpu_offload: bool = Field(default_factory=_default_cpu_offload)
|
||||
"""Store embedding weights in pinned CPU memory for UVA lookup.
|
||||
Defaults to False, or VLLM_PLE_CPU_OFFLOAD when set for compatibility.
|
||||
An explicit value takes precedence over the legacy environment variable."""
|
||||
|
||||
embedding_across_dp: bool = False
|
||||
"""Shard embeddings across TP and all DP ranks when enabled.
|
||||
Otherwise, each DP rank has a separate TP-sharded embedding replica."""
|
||||
|
||||
def verify_model_config(self, model_config: "ModelConfig | None") -> None:
|
||||
"""Reject Engram configuration for models without supported embeddings."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
supported_architectures = {
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
}
|
||||
if (
|
||||
model_config is None
|
||||
or model_config.architecture not in supported_architectures
|
||||
or not current_platform.is_cuda()
|
||||
or not getattr(model_config.hf_text_config, "ple_layer_ids", None)
|
||||
):
|
||||
raise ValueError(
|
||||
"EngramConfig requires a model with supported Engram "
|
||||
"embeddings. Currently only the CUDA Qwen4Exp implementation "
|
||||
"with non-empty ple_layer_ids is supported."
|
||||
)
|
||||
|
||||
def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
|
||||
"""Reject unsupported embedding parallel topologies."""
|
||||
if (
|
||||
self.embedding_across_dp
|
||||
and parallel_config.data_parallel_size > 1
|
||||
and parallel_config.enable_elastic_ep
|
||||
):
|
||||
raise ValueError(
|
||||
"Engram embedding_across_dp is not supported with elastic EP yet."
|
||||
)
|
||||
|
||||
def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
|
||||
"""Derive the embedding group size from the parallel configuration."""
|
||||
size = parallel_config.tensor_parallel_size
|
||||
if self.embedding_across_dp and parallel_config.data_parallel_size > 1:
|
||||
size *= parallel_config.data_parallel_size
|
||||
return size
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""Hash settings that affect embedding execution and graph structure."""
|
||||
return hash_factors(get_hash_factors(self, set()))
|
||||
@@ -34,6 +34,7 @@ from .device import DeviceConfig
|
||||
from .diffusion import DiffusionConfig
|
||||
from .ec_manager_config import EncoderCacheManagerConfig
|
||||
from .ec_transfer import ECTransferConfig
|
||||
from .engram import EngramConfig
|
||||
from .kernel import KernelConfig
|
||||
from .kv_events import KVEventsConfig
|
||||
from .kv_transfer import KVTransferConfig
|
||||
@@ -370,6 +371,8 @@ class VllmConfig:
|
||||
"""Model weight offloading configuration."""
|
||||
attention_config: AttentionConfig = Field(default_factory=AttentionConfig)
|
||||
"""Attention configuration."""
|
||||
engram_config: EngramConfig | None = None
|
||||
"""Optional Engram configuration, only valid for supported PLE models."""
|
||||
mamba_config: MambaConfig = Field(default_factory=MambaConfig)
|
||||
"""Mamba configuration."""
|
||||
kernel_config: KernelConfig = Field(default_factory=KernelConfig)
|
||||
@@ -506,6 +509,11 @@ class VllmConfig:
|
||||
vllm_factors.append(self.attention_config.compute_hash())
|
||||
else:
|
||||
vllm_factors.append("None")
|
||||
vllm_factors.append(
|
||||
self.engram_config.compute_hash()
|
||||
if self.engram_config is not None
|
||||
else "None"
|
||||
)
|
||||
if self.lora_config:
|
||||
vllm_factors.append(self.lora_config.compute_hash())
|
||||
else:
|
||||
@@ -1083,6 +1091,25 @@ class VllmConfig:
|
||||
if not self.use_v2_model_runner:
|
||||
raise ValueError("trace replay requires Model Runner V2")
|
||||
|
||||
def _resolve_and_verify_engram_config(self) -> None:
|
||||
"""Resolve legacy offload settings and validate model and parallel configs."""
|
||||
if self.engram_config is None:
|
||||
if not envs.VLLM_PLE_CPU_OFFLOAD:
|
||||
return
|
||||
self.engram_config = EngramConfig()
|
||||
model_config = self.model_config
|
||||
speculative_config = self.speculative_config
|
||||
# Draft configs inherit the target's communication groups and settings.
|
||||
# Qwen4Exp MTP itself disables PLE, so validate its target instead.
|
||||
if (
|
||||
speculative_config is not None
|
||||
and model_config is speculative_config.draft_model_config
|
||||
):
|
||||
model_config = speculative_config.target_model_config
|
||||
self.engram_config.verify_model_config(model_config)
|
||||
self.engram_config.verify_parallel_config(self.parallel_config)
|
||||
logger.info_once("Resolved Engram configuration: %s", str(self.engram_config))
|
||||
|
||||
def __post_init__(self):
|
||||
"""Verify configs are valid & consistent with each other."""
|
||||
|
||||
@@ -1095,6 +1122,7 @@ class VllmConfig:
|
||||
logger.info_once("Performance mode set to '%s'.", self.performance_mode)
|
||||
|
||||
self.try_verify_and_update_config()
|
||||
self._resolve_and_verify_engram_config()
|
||||
|
||||
# Models may have supplied their own DCP defaults above; anything still
|
||||
# unset falls back to the stock ones.
|
||||
|
||||
@@ -47,7 +47,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
global_world_size,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
if "tp" not in unique_name:
|
||||
# Match the group name exactly so ETP does not enable TP-only backends.
|
||||
if unique_name.split(":")[0] != "tp":
|
||||
# custom allreduce or torch symm mem can be used only by tp
|
||||
use_custom_allreduce = False
|
||||
use_torch_symm_mem = False
|
||||
|
||||
@@ -1551,6 +1551,15 @@ def get_tp_group() -> GroupCoordinator:
|
||||
return _TP
|
||||
|
||||
|
||||
_ETP: GroupCoordinator | None = None
|
||||
|
||||
|
||||
def get_etp_group() -> GroupCoordinator:
|
||||
"""Return the Engram Tensor Parallel (ETP) group that shards one embedding table."""
|
||||
assert _ETP is not None, "Engram tensor-parallel group is not initialized"
|
||||
return _ETP
|
||||
|
||||
|
||||
_DCP: GroupCoordinator | None = None
|
||||
|
||||
|
||||
@@ -2008,6 +2017,28 @@ def initialize_model_parallel(
|
||||
group_name="tp",
|
||||
)
|
||||
|
||||
global _ETP
|
||||
assert _ETP is None, "Engram tensor-parallel group is already initialized"
|
||||
engram_tensor_parallel_size = (
|
||||
config.engram_config.get_parallel_size(parallel_config)
|
||||
if config.engram_config is not None
|
||||
else tensor_model_parallel_size
|
||||
)
|
||||
if engram_tensor_parallel_size == tensor_model_parallel_size:
|
||||
_ETP = _TP
|
||||
else:
|
||||
group_ranks = (
|
||||
all_ranks.permute(0, 2, 3, 1, 4)
|
||||
.reshape(-1, engram_tensor_parallel_size)
|
||||
.unbind(0)
|
||||
)
|
||||
_ETP = init_model_parallel_group(
|
||||
[ranks.tolist() for ranks in group_ranks],
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
group_name="etp",
|
||||
)
|
||||
|
||||
# Build the DCP model-parallel groups.
|
||||
global _DCP
|
||||
assert _DCP is None, "decode context model parallel group is already initialized"
|
||||
@@ -2142,13 +2173,14 @@ def initialize_model_parallel(
|
||||
logger.info_once(
|
||||
"rank %s in world size %s is assigned as "
|
||||
"DP rank %s, PP rank %s, PCP rank %s, "
|
||||
"TP rank %s, EP rank %s, EPLB rank %s",
|
||||
"TP rank %s, ETP rank %s, EP rank %s, EPLB rank %s",
|
||||
rank,
|
||||
world_size,
|
||||
_DP.rank_in_group,
|
||||
_PP.rank_in_group,
|
||||
_PCP.rank_in_group,
|
||||
_TP.rank_in_group,
|
||||
_ETP.rank_in_group,
|
||||
_EP.rank_in_group if _EP is not None else "N/A",
|
||||
_EPLB.rank_in_group if _EPLB is not None else "N/A",
|
||||
)
|
||||
@@ -2243,7 +2275,11 @@ def get_node_count() -> int:
|
||||
|
||||
def destroy_model_parallel():
|
||||
"""Set the groups to none and destroy them."""
|
||||
global _TP
|
||||
global _TP, _ETP
|
||||
|
||||
if _ETP and _ETP is not _TP:
|
||||
_ETP.destroy()
|
||||
_ETP = None
|
||||
|
||||
if _TP:
|
||||
_TP.destroy()
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.config import (
|
||||
DiffusionConfig,
|
||||
ECTransferConfig,
|
||||
EncoderCacheManagerConfig,
|
||||
EngramConfig,
|
||||
EPLBConfig,
|
||||
FaultToleranceConfig,
|
||||
KernelConfig,
|
||||
@@ -688,6 +689,7 @@ class EngineArgs:
|
||||
pooler_config: PoolerConfig | None = ModelConfig.pooler_config
|
||||
compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
|
||||
attention_config: AttentionConfig = get_field(VllmConfig, "attention_config")
|
||||
engram_config: EngramConfig | None = VllmConfig.engram_config
|
||||
mamba_config: MambaConfig = get_field(VllmConfig, "mamba_config")
|
||||
kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config")
|
||||
enable_flashinfer_autotune: bool = get_field(
|
||||
@@ -788,6 +790,8 @@ class EngineArgs:
|
||||
self.compilation_config = CompilationConfig(**self.compilation_config)
|
||||
if isinstance(self.attention_config, dict):
|
||||
self.attention_config = AttentionConfig(**self.attention_config)
|
||||
if isinstance(self.engram_config, dict):
|
||||
self.engram_config = EngramConfig(**self.engram_config)
|
||||
if isinstance(self.mamba_config, dict):
|
||||
self.mamba_config = MambaConfig(**self.mamba_config)
|
||||
if isinstance(self.kernel_config, dict):
|
||||
@@ -1701,6 +1705,7 @@ class EngineArgs:
|
||||
vllm_group.add_argument(
|
||||
"--attention-config", "-ac", **vllm_kwargs["attention_config"]
|
||||
)
|
||||
vllm_group.add_argument("--engram-config", **vllm_kwargs["engram_config"])
|
||||
vllm_group.add_argument("--reasoning-config", **vllm_kwargs["reasoning_config"])
|
||||
vllm_group.add_argument("--kernel-config", **vllm_kwargs["kernel_config"])
|
||||
vllm_group.add_argument(
|
||||
@@ -2599,6 +2604,7 @@ class EngineArgs:
|
||||
load_config=load_config,
|
||||
offload_config=offload_config,
|
||||
attention_config=attention_config,
|
||||
engram_config=self.engram_config,
|
||||
mamba_config=mamba_config,
|
||||
kernel_config=kernel_config,
|
||||
lora_config=lora_config,
|
||||
|
||||
@@ -155,6 +155,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: bool = False
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
|
||||
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
|
||||
VLLM_PLE_CPU_OFFLOAD: bool = False
|
||||
VLLM_DISABLE_COMPILE_CACHE: bool = False
|
||||
VLLM_REPLICATE_EMBED: bool = False
|
||||
VLLM_USE_LAYERNAME: bool = True
|
||||
@@ -2077,6 +2078,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_LOG_MODEL_INSPECTION": lambda: bool(
|
||||
int(os.getenv("VLLM_LOG_MODEL_INSPECTION", "0"))
|
||||
),
|
||||
# Keep Qwen4Exp PLE embedding tables in pinned CPU memory and gather their
|
||||
# rows through UVA on a dedicated CUDA stream.
|
||||
# Legacy fallback for EngramConfig.cpu_offload, which takes precedence.
|
||||
# This environment variable may be removed in a future release.
|
||||
"VLLM_PLE_CPU_OFFLOAD": lambda: bool(int(os.getenv("VLLM_PLE_CPU_OFFLOAD", "0"))),
|
||||
# Debug logging for --enable-mfu-metrics
|
||||
"VLLM_DEBUG_MFU_METRICS": lambda: bool(
|
||||
int(os.getenv("VLLM_DEBUG_MFU_METRICS", "0"))
|
||||
|
||||
@@ -23,6 +23,9 @@ else:
|
||||
class QuantizeMethodBase(ABC):
|
||||
"""Base class for different quantized methods."""
|
||||
|
||||
requires_device_loading: bool = True
|
||||
"""Whether post-load processing requires parameters on the target device."""
|
||||
|
||||
uses_meta_device: bool = False
|
||||
"""
|
||||
Whether this method creates weights on meta device for online quantization.
|
||||
|
||||
@@ -11,6 +11,7 @@ from torch.nn.parameter import Parameter
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed import (
|
||||
GroupCoordinator,
|
||||
divide,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -240,6 +241,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
prefix: full name of the layer in the state dict
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
quant_method: Preselected quantization method for model-specific layers.
|
||||
parallel_group: Process group used to shard and reduce the embedding.
|
||||
""" # noqa: E501
|
||||
|
||||
# --8<-- [end:vocab_parallel_embedding]
|
||||
@@ -256,13 +258,18 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
quant_method: QuantizeMethodBase | None = None,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# Keep the input dimensions.
|
||||
self.disable_tp = disable_tp
|
||||
self.parallel_group = parallel_group
|
||||
if disable_tp:
|
||||
tp_rank, self.tp_size = 0, 1
|
||||
elif parallel_group is not None:
|
||||
tp_rank = parallel_group.rank_in_group
|
||||
self.tp_size = parallel_group.world_size
|
||||
else:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
@@ -535,10 +542,16 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
|
||||
comm_output = output_parallel.view(torch.int8)
|
||||
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
output = tensor_model_parallel_all_reduce(comm_output)
|
||||
output = (
|
||||
self.parallel_group.all_reduce(comm_output)
|
||||
if self.parallel_group is not None
|
||||
else tensor_model_parallel_all_reduce(comm_output)
|
||||
)
|
||||
return output.view(output_parallel.dtype)
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
# Reduce across all the model parallel GPUs.
|
||||
if self.parallel_group is not None:
|
||||
return self.parallel_group.all_reduce(output_parallel)
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import inspect
|
||||
import warnings
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -120,11 +120,18 @@ def process_weights_after_loading(
|
||||
"does not support pre-processed weights"
|
||||
)
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
# (for repacking, quantizing, etc), they typically expect parameters
|
||||
# to be on the global target device. This scope is for the
|
||||
# case where cpu offloading is used, where we will move the
|
||||
# parameters onto device for processing and back off after.
|
||||
with device_loading_context(module, target_device):
|
||||
# Methods that can process weights in place (e.g. PLE scale
|
||||
# validation) set requires_device_loading=False to skip this move.
|
||||
loading_context = (
|
||||
device_loading_context(module, target_device)
|
||||
if quant_method.requires_device_loading
|
||||
else nullcontext()
|
||||
)
|
||||
with loading_context:
|
||||
quant_method.process_weights_after_loading(module)
|
||||
# process_weights_after_loading may swap in freshly-created
|
||||
# Parameters (e.g. FP8 requantization), which are stamped with the
|
||||
|
||||
@@ -461,6 +461,27 @@ class Qwen4ExpModel(nn.Module):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
@staticmethod
|
||||
def _start_layer_ple_prefetch(
|
||||
layer: nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
ngram_context: torch.Tensor | None,
|
||||
) -> None:
|
||||
"""Start a layer's PLE prefetch when the required inputs exist."""
|
||||
ple: Qwen4ExpPLELayer | None = getattr(layer, "ple", None)
|
||||
if ple is None:
|
||||
return
|
||||
if input_ids is None or query_start_loc is None or ngram_context is None:
|
||||
raise RuntimeError("PLE inputs were not prepared")
|
||||
ple.start_prefetch(
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
@@ -487,10 +508,26 @@ class Qwen4ExpModel(nn.Module):
|
||||
block_output = None
|
||||
injection = None
|
||||
last_layer = None
|
||||
if self.start_layer < self.end_layer:
|
||||
self._start_layer_ple_prefetch(
|
||||
self.layers[self.start_layer],
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
for layer_idx, layer in islice(
|
||||
enumerate(self.layers), self.start_layer, self.end_layer
|
||||
):
|
||||
last_layer = layer
|
||||
if layer_idx + 1 < self.end_layer:
|
||||
self._start_layer_ple_prefetch(
|
||||
self.layers[layer_idx + 1],
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
hidden_states, block_output, injection = layer(
|
||||
hidden_states=hidden_states,
|
||||
prev_block_output=block_output,
|
||||
|
||||
@@ -0,0 +1,939 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp n-gram embeddings with device and pinned-host storage."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_dp_group, get_etp_group, get_tp_group
|
||||
from vllm.forward_context import DPMetadata, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptQuantConfigBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.model_executor.parameter import (
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
|
||||
from ..common.ple import PLEVocabParallelEmbedding
|
||||
from .ops.ple import ple_ngram_ids
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||
"""ETP-sharded PLE table shared by device and pinned-host backends."""
|
||||
|
||||
supports_prefetch: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
*,
|
||||
params_dtype: torch.dtype,
|
||||
padding_size: int,
|
||||
prefix: str,
|
||||
embedding_method: "Qwen4ExpPLEEmbeddingMethod",
|
||||
num_ngram_heads: int = 1,
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
) -> None:
|
||||
del num_ngram_heads, max_total_tokens
|
||||
super().__init__(
|
||||
num_embeddings,
|
||||
embedding_dim,
|
||||
params_dtype=params_dtype,
|
||||
padding_size=padding_size,
|
||||
prefix=prefix,
|
||||
quant_method=embedding_method,
|
||||
parallel_group=get_etp_group(),
|
||||
)
|
||||
self.embedding_method = embedding_method
|
||||
self.data_parallel_rank = data_parallel_rank
|
||||
tp_size = get_tp_group().world_size
|
||||
if self.tp_size % tp_size:
|
||||
raise ValueError(
|
||||
"ETP size must be divisible by TP size, but got "
|
||||
f"ETP={self.tp_size} and TP={tp_size}"
|
||||
)
|
||||
self.etp_data_parallel_size = self.tp_size // tp_size
|
||||
|
||||
@abstractmethod
|
||||
def allocate_embedding_weight(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Allocate storage for the complete embedding weight."""
|
||||
raise NotImplementedError
|
||||
|
||||
def dequantize(
|
||||
self,
|
||||
embeddings: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Delegate storage-format conversion to the embedding method."""
|
||||
return self.embedding_method.dequantize(self, embeddings, output_dtype)
|
||||
|
||||
def _get_dp_gather_slot(self, local_num_tokens: int) -> tuple[int, int]:
|
||||
"""Return the per-DP slot size and this rank's slot offset."""
|
||||
if self.etp_data_parallel_size == 1:
|
||||
return local_num_tokens, 0
|
||||
dp_metadata: DPMetadata | None = get_forward_context().dp_metadata
|
||||
if dp_metadata is None:
|
||||
raise RuntimeError("ETP spanning DP requires DP token metadata")
|
||||
group_start = (self.data_parallel_rank // self.etp_data_parallel_size) * (
|
||||
self.etp_data_parallel_size
|
||||
)
|
||||
group_end = group_start + self.etp_data_parallel_size
|
||||
token_counts = dp_metadata.num_tokens_across_dp_cpu.tolist()
|
||||
group_counts = token_counts[group_start:group_end]
|
||||
slot_size = max(group_counts)
|
||||
dp_rank = get_dp_group().rank_in_group
|
||||
return slot_size, dp_rank * slot_size
|
||||
|
||||
def _gather_dp_ids(
|
||||
self,
|
||||
ngram_ids: torch.Tensor,
|
||||
slot_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Gather DP-local IDs that share one ETP-sharded PLE table."""
|
||||
if self.etp_data_parallel_size == 1:
|
||||
return ngram_ids
|
||||
if ngram_ids.shape[0] < slot_size:
|
||||
padding = ngram_ids.new_zeros(
|
||||
slot_size - ngram_ids.shape[0], ngram_ids.shape[1]
|
||||
)
|
||||
ngram_ids = torch.cat((ngram_ids, padding), dim=0)
|
||||
return get_dp_group().all_gather(ngram_ids, dim=0)
|
||||
|
||||
def _select_embeddings(
|
||||
self,
|
||||
embeddings: torch.Tensor,
|
||||
local_num_tokens: int,
|
||||
slot_offset: int,
|
||||
) -> torch.Tensor:
|
||||
"""Select this DP rank's rows from the ETP-reduced embeddings."""
|
||||
if self.etp_data_parallel_size == 1:
|
||||
return embeddings
|
||||
return embeddings.narrow(0, slot_offset, local_num_tokens)
|
||||
|
||||
@abstractmethod
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
"""Start an asynchronous lookup when supported."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
"""Quantization interface shared by resident and pinned PLE tables."""
|
||||
|
||||
# PLE post-load processing only validates scales in their current storage.
|
||||
requires_device_loading: bool = False
|
||||
|
||||
@staticmethod
|
||||
def from_quant_config(
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
embedding_dtype: str | None = None,
|
||||
) -> "Qwen4ExpPLEEmbeddingMethod":
|
||||
"""Select the concrete PLE embedding format for a layer."""
|
||||
if embedding_dtype == "float8_e4m3fn":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if quant_config is None:
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
|
||||
if quant_config._resolve_quant_algo(prefix) == "FP8":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if isinstance(
|
||||
quant_config, ModelOptQuantConfigBase
|
||||
) and quant_config.is_layer_excluded(prefix):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if not isinstance(quant_config, Fp8Config):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE embedding does not support quantization config "
|
||||
f"{type(quant_config).__name__}"
|
||||
)
|
||||
|
||||
ignored_layers = quant_config.ignored_layers
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
ignored_layers,
|
||||
quant_config.packed_modules_mapping,
|
||||
match_mode=quant_config.ignored_layers_match_mode,
|
||||
):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
# PLE checkpoint shards form one runtime embedding parameter.
|
||||
shard_prefix = f"{prefix}.shard_"
|
||||
if any(name.startswith(shard_prefix) for name in ignored_layers):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if not quant_config.is_checkpoint_fp8_serialized:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE embedding only supports serialized FP8 checkpoints"
|
||||
)
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError("PLE weights only support embedding lookup")
|
||||
|
||||
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
|
||||
return F.embedding(input_, layer.weight)
|
||||
|
||||
@abstractmethod
|
||||
def dequantize(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
embeddings: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Convert looked-up PLE rows to the activation dtype."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Qwen4ExpPLEUnquantizedEmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
|
||||
"""Unquantized PLE embedding storage and lookup semantics."""
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: Qwen4ExpPLEEmbedding,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size
|
||||
weight = nn.Parameter(
|
||||
layer.allocate_embedding_weight(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
def dequantize(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
embeddings: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
del layer, output_dtype
|
||||
return embeddings
|
||||
|
||||
|
||||
class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
|
||||
"""FP8 PLE embedding with one global checkpoint scale."""
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: Qwen4ExpPLEEmbedding,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size, params_dtype
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
weight = ModelWeightParameter(
|
||||
data=layer.allocate_embedding_weight(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
torch.float8_e4m3fn,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
weight_scale = create_fp8_scale_parameter(
|
||||
PerTensorScaleParameter,
|
||||
output_partition_sizes,
|
||||
input_size_per_partition,
|
||||
None,
|
||||
weight_loader,
|
||||
scale_dtype=torch.float32,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: nn.Module) -> None:
|
||||
"""Reject FP8 PLE checkpoints without a global scale."""
|
||||
sentinel = torch.finfo(torch.float32).min
|
||||
if torch.any(layer.weight_scale == sentinel):
|
||||
raise ValueError("FP8 PLE checkpoint is missing its global scale")
|
||||
|
||||
def dequantize(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
embeddings: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
weight_scale = getattr(layer, "weight_scale", None)
|
||||
if weight_scale is None:
|
||||
raise RuntimeError("FP8 PLE embedding is missing its global scale")
|
||||
if weight_scale.device != embeddings.device:
|
||||
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
|
||||
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
|
||||
|
||||
|
||||
class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
|
||||
"""PLE table allocated on the active model device."""
|
||||
|
||||
def allocate_embedding_weight(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Allocate the complete PLE weight on the active device."""
|
||||
return torch.empty(num_embeddings, embedding_dim, dtype=dtype)
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
"""Resident embedding prefetch is a no-op."""
|
||||
return None
|
||||
|
||||
def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Gather ETP inputs, look up embeddings, and select local rows."""
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
|
||||
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
|
||||
embeddings = super().forward(gathered_ids)
|
||||
return self._select_embeddings(
|
||||
embeddings,
|
||||
ngram_ids.shape[0],
|
||||
slot_offset,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _lookup_ple_embedding_from_pinned_kernel(
|
||||
weight_ptr,
|
||||
ids_ptr,
|
||||
output_ptr,
|
||||
embedding_dim,
|
||||
tp_vocab_start,
|
||||
tp_vocab_end,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Look up TP-owned PLE rows through a CUDA view of pinned host memory."""
|
||||
row_id = tl.program_id(0)
|
||||
global_idx = tl.load(ids_ptr + row_id)
|
||||
in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end)
|
||||
local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0)
|
||||
offsets = tl.arange(0, BLOCK_D)
|
||||
store_mask = offsets < embedding_dim
|
||||
load_mask = store_mask & in_range
|
||||
values = tl.load(
|
||||
weight_ptr + local_idx * embedding_dim + offsets,
|
||||
mask=load_mask,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(
|
||||
output_ptr + row_id * embedding_dim + offsets,
|
||||
values,
|
||||
mask=store_mask,
|
||||
)
|
||||
|
||||
|
||||
class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
"""PLE table loaded into pinned CPU memory and looked up through UVA."""
|
||||
|
||||
supports_prefetch: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
*,
|
||||
params_dtype: torch.dtype,
|
||||
padding_size: int,
|
||||
prefix: str,
|
||||
embedding_method: Qwen4ExpPLEEmbeddingMethod,
|
||||
num_ngram_heads: int = 1,
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
) -> None:
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
super().__init__(
|
||||
num_embeddings,
|
||||
embedding_dim,
|
||||
params_dtype=params_dtype,
|
||||
padding_size=padding_size,
|
||||
prefix=prefix,
|
||||
embedding_method=embedding_method,
|
||||
num_ngram_heads=num_ngram_heads,
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
|
||||
self._block_d = triton.next_power_of_2(self.embedding_dim)
|
||||
self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device)
|
||||
self._prefetch_buffer = torch.empty(
|
||||
max_total_tokens * self.etp_data_parallel_size,
|
||||
num_ngram_heads,
|
||||
self.embedding_dim,
|
||||
dtype=self.weight.dtype,
|
||||
device=self._uva_weight.device,
|
||||
)
|
||||
self._output_dim = num_ngram_heads * self.embedding_dim
|
||||
|
||||
def allocate_embedding_weight(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Allocate the complete PLE weight directly in pinned CPU memory."""
|
||||
return torch.empty(
|
||||
num_embeddings,
|
||||
embedding_dim,
|
||||
dtype=dtype,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
|
||||
def _lookup(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Look up local ETP rows while preserving the weight storage dtype."""
|
||||
expected_shape = (*input_ids.shape, self.embedding_dim)
|
||||
if output is None:
|
||||
output = torch.empty(
|
||||
expected_shape,
|
||||
dtype=self.weight.dtype,
|
||||
device=input_ids.device,
|
||||
)
|
||||
elif (
|
||||
tuple(output.shape) != expected_shape
|
||||
or output.dtype != self.weight.dtype
|
||||
or output.device != input_ids.device
|
||||
):
|
||||
raise ValueError(
|
||||
"PLE prefetch output must match the input shape, weight dtype, "
|
||||
"and input device"
|
||||
)
|
||||
|
||||
flat_ids = input_ids.reshape(-1).long()
|
||||
if flat_ids.numel():
|
||||
_lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)](
|
||||
self._uva_weight,
|
||||
flat_ids,
|
||||
output,
|
||||
self.embedding_dim,
|
||||
self.shard_indices.org_vocab_start_index,
|
||||
self.shard_indices.org_vocab_end_index,
|
||||
BLOCK_D=self._block_d,
|
||||
)
|
||||
return output
|
||||
|
||||
def _reduce_etp_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
|
||||
"""Combine pinned lookup results owned by different ETP ranks."""
|
||||
if self.tp_size == 1:
|
||||
return embeddings
|
||||
assert self.parallel_group is not None
|
||||
if embeddings.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
|
||||
# Each vocabulary row has one owner, so reduce the raw FP8 bytes.
|
||||
reduced = self.parallel_group.all_reduce(embeddings.view(torch.int8))
|
||||
return reduced.view(embeddings.dtype)
|
||||
return self.parallel_group.all_reduce(embeddings)
|
||||
|
||||
@eager_break_during_capture
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
"""Gather ETP IDs and launch their UVA lookup on the side stream."""
|
||||
slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0])
|
||||
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
|
||||
active_output = self._prefetch_buffer[: gathered_ids.shape[0]]
|
||||
prefetch_stream = self._prefetch_stream
|
||||
prefetch_stream.wait_stream(torch.cuda.current_stream())
|
||||
gathered_ids.record_stream(prefetch_stream)
|
||||
with torch.cuda.stream(prefetch_stream):
|
||||
self._lookup(gathered_ids, output=active_output)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _finalize_prefetch(
|
||||
self,
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
"""Join the side stream, reduce ETP shards, and select local rows."""
|
||||
torch.cuda.current_stream().wait_stream(self._prefetch_stream)
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0])
|
||||
active_output = prefetch_output[: slot_size * self.etp_data_parallel_size]
|
||||
embeddings = self._reduce_etp_embeddings(active_output)
|
||||
embeddings = self._select_embeddings(
|
||||
embeddings,
|
||||
output.shape[0],
|
||||
slot_offset,
|
||||
)
|
||||
output.copy_(embeddings.flatten(-2))
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Finish the pinned lookup into graph-owned output storage."""
|
||||
output = self._prefetch_buffer.new_empty(
|
||||
(hidden_states.shape[0], self._output_dim)
|
||||
)
|
||||
self._finalize_prefetch(self._prefetch_buffer, output)
|
||||
return output
|
||||
|
||||
|
||||
class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
_MASK64 = (1 << 64) - 1
|
||||
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
|
||||
_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
|
||||
_SPLITMIX_M2 = 0x94D049BB133111EB
|
||||
_PLE_LAYER_PRIME = 10007
|
||||
|
||||
@classmethod
|
||||
def _splitmix64(cls, value: int) -> int:
|
||||
"""Mix an integer into a deterministic unsigned 64-bit value."""
|
||||
value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64
|
||||
value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64
|
||||
value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64
|
||||
return (value ^ (value >> 31)) & cls._MASK64
|
||||
|
||||
@staticmethod
|
||||
def _is_prime_64(value: int) -> bool:
|
||||
"""Return whether a 64-bit integer is prime."""
|
||||
if value < 2:
|
||||
return False
|
||||
for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
|
||||
if value % prime == 0:
|
||||
return value == prime
|
||||
exponent = value - 1
|
||||
shifts = 0
|
||||
while exponent % 2 == 0:
|
||||
exponent //= 2
|
||||
shifts += 1
|
||||
for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022):
|
||||
if base % value == 0:
|
||||
continue
|
||||
witness = pow(base, exponent, value)
|
||||
if witness in (1, value - 1):
|
||||
continue
|
||||
for _ in range(shifts - 1):
|
||||
witness = pow(witness, 2, value)
|
||||
if witness == value - 1:
|
||||
break
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _nth_prime_after(cls, start: int, count: int) -> int:
|
||||
"""Return the ``count``-th prime strictly greater than ``start``."""
|
||||
prime = int(start)
|
||||
for _ in range(count):
|
||||
candidate = prime + 1
|
||||
if candidate <= 2:
|
||||
prime = 2
|
||||
continue
|
||||
if candidate % 2 == 0:
|
||||
candidate += 1
|
||||
while not cls._is_prime_64(candidate):
|
||||
candidate += 2
|
||||
prime = candidate
|
||||
return prime
|
||||
|
||||
@classmethod
|
||||
def _make_layer_multipliers(
|
||||
cls,
|
||||
*,
|
||||
ngram_size: int,
|
||||
unigram_vocab_size: int,
|
||||
seed: int,
|
||||
ple_dense_layer_id: int,
|
||||
) -> list[int]:
|
||||
"""Build deterministic hash multipliers for one PLE layer."""
|
||||
max_multiplier = ((1 << 63) - 1) // unigram_vocab_size
|
||||
half_bound = max(1, max_multiplier // 2)
|
||||
base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id
|
||||
multipliers = []
|
||||
for index in range(ngram_size):
|
||||
value = base_seed + cls._SPLITMIX_GAMMA * (index + 1)
|
||||
multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1)
|
||||
return multipliers
|
||||
|
||||
@classmethod
|
||||
def _make_vocab_layout(
|
||||
cls,
|
||||
*,
|
||||
ngram_vocab_size_base: int,
|
||||
ngram_heads: int,
|
||||
ple_dense_layer_id: int,
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
"""Build per-head vocabulary sizes, offsets, and total row count."""
|
||||
sizes: list[int] = []
|
||||
offsets: list[int] = []
|
||||
offset = 0
|
||||
for local_head in range(ngram_heads):
|
||||
global_head = ple_dense_layer_id * ngram_heads + local_head
|
||||
size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
|
||||
sizes.append(size)
|
||||
offsets.append(offset)
|
||||
offset += size
|
||||
return sizes, offsets, offset
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
max_total_tokens: int,
|
||||
*,
|
||||
data_parallel_rank: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.embedding_dim = embedding_dim
|
||||
self.ngram_size = int(config.ngram_size)
|
||||
self.heads_per_ngram = int(config.heads_per_ngram)
|
||||
self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
|
||||
if self.ngram_size < 2:
|
||||
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
|
||||
if self.heads_per_ngram <= 0:
|
||||
raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}")
|
||||
if embedding_dim % self.ngram_heads:
|
||||
raise ValueError(
|
||||
"ple_embed_dim must be divisible by total ngram heads: "
|
||||
f"{embedding_dim} % {self.ngram_heads} != 0"
|
||||
)
|
||||
self.head_dim = embedding_dim // self.ngram_heads
|
||||
self.eos_token_id = int(config.eos_token_id)
|
||||
self.unigram_vocab_size = int(config.vocab_size)
|
||||
self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512))
|
||||
if self.split_ngram_parts <= 0:
|
||||
raise ValueError("split_ngram_parts must be positive")
|
||||
|
||||
multipliers = self._make_layer_multipliers(
|
||||
ngram_size=self.ngram_size,
|
||||
unigram_vocab_size=self.unigram_vocab_size,
|
||||
seed=int(getattr(config, "seed", 1234)),
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
)
|
||||
self.register_buffer(
|
||||
"layer_multipliers",
|
||||
torch.tensor(multipliers, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
sizes, offsets, total_vocab_size = self._make_vocab_layout(
|
||||
ngram_vocab_size_base=int(config.ngram_vocab_size_base),
|
||||
ngram_heads=self.ngram_heads,
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
)
|
||||
self.register_buffer(
|
||||
"ngram_heads_vocab_sizes",
|
||||
torch.tensor(sizes, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
self.register_buffer(
|
||||
"ngram_heads_offsets",
|
||||
torch.tensor(offsets, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
divisor = int(config.make_ngram_vocab_size_divisible_by)
|
||||
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
|
||||
embedding_prefix = f"{prefix}.ngram_embedding"
|
||||
embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
embedding_prefix,
|
||||
getattr(config, "ple_embedding_dtype", None),
|
||||
)
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
embedding_cls = (
|
||||
Qwen4ExpPLEPinnedHostEmbedding
|
||||
if engram_config is not None and engram_config.cpu_offload
|
||||
else Qwen4ExpPLEDeviceEmbedding
|
||||
)
|
||||
self.ngram_embedding = embedding_cls(
|
||||
padded_vocab_size,
|
||||
self.head_dim,
|
||||
params_dtype=params_dtype,
|
||||
padding_size=divisor,
|
||||
prefix=embedding_prefix,
|
||||
embedding_method=embedding_quant_method,
|
||||
num_ngram_heads=self.ngram_heads,
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
weight = self.ngram_embedding.weight
|
||||
logger.info(
|
||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s",
|
||||
embedding_prefix,
|
||||
type(embedding_quant_method).__name__,
|
||||
weight.dtype,
|
||||
weight.device,
|
||||
weight.is_pinned(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _shift_precompute(
|
||||
tokens: torch.Tensor, eos_token_id: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if tokens.dim() != 2:
|
||||
raise ValueError("tokens must be a 2D tensor")
|
||||
batch_size, seq_len = tokens.shape
|
||||
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
|
||||
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
|
||||
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
|
||||
previous_eos = torch.cat(
|
||||
[
|
||||
eos_positions.new_full((batch_size, 1), -1),
|
||||
previous_eos_inclusive[:, :-1],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
return positions, positions.unsqueeze(0) - previous_eos - 1
|
||||
|
||||
@staticmethod
|
||||
def _shift_apply(
|
||||
tokens: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
position_in_segment: torch.Tensor,
|
||||
shift: int,
|
||||
eos_token_id: int,
|
||||
) -> torch.Tensor:
|
||||
if shift == 0:
|
||||
return tokens
|
||||
source = positions - shift
|
||||
gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1)
|
||||
shifted = tokens.gather(1, gather_indices)
|
||||
valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift)
|
||||
return torch.where(valid, shifted, tokens.new_full((), eos_token_id))
|
||||
|
||||
def compute_ngram_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute n-gram embedding indices for the current request layout."""
|
||||
input_ids = input_ids.reshape(-1)
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
num_tokens = input_ids.shape[0]
|
||||
|
||||
if input_ids.is_cuda:
|
||||
return ple_ngram_ids(
|
||||
input_ids=input_ids,
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=ngram_context,
|
||||
layer_multipliers=self.layer_multipliers,
|
||||
ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes,
|
||||
ngram_heads_offsets=self.ngram_heads_offsets,
|
||||
eos_token_id=self.eos_token_id,
|
||||
heads_per_ngram=self.heads_per_ngram,
|
||||
output=output,
|
||||
)
|
||||
input_ids = input_ids.long()
|
||||
query_start_loc = query_start_loc.long()
|
||||
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
|
||||
packed = torch.full(
|
||||
(num_reqs, num_tokens),
|
||||
self.eos_token_id,
|
||||
device=input_ids.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
|
||||
request_indices.clamp_(max=num_reqs - 1)
|
||||
columns = (positions - query_start_loc[request_indices]).clamp(
|
||||
0, packed.shape[1] - 1
|
||||
)
|
||||
packed[request_indices, columns] = input_ids
|
||||
ngram_context = ngram_context[:num_reqs].to(
|
||||
device=input_ids.device, dtype=torch.long
|
||||
)
|
||||
|
||||
context = torch.cat([ngram_context, packed], dim=-1)
|
||||
positions_2d, position_in_segment = self._shift_precompute(
|
||||
context, self.eos_token_id
|
||||
)
|
||||
shifted = [context]
|
||||
for shift in range(1, self.ngram_size):
|
||||
shifted.append(
|
||||
self._shift_apply(
|
||||
context,
|
||||
positions_2d,
|
||||
position_in_segment,
|
||||
shift,
|
||||
self.eos_token_id,
|
||||
)
|
||||
)
|
||||
adjusted_columns = columns + self.ngram_size - 1
|
||||
id_blocks = []
|
||||
for ngram in range(2, self.ngram_size + 1):
|
||||
start = (ngram - 2) * self.heads_per_ngram
|
||||
end = start + self.heads_per_ngram
|
||||
mixed = shifted[0] * self.layer_multipliers[0]
|
||||
for index in range(1, ngram):
|
||||
mixed = torch.bitwise_xor(
|
||||
mixed, shifted[index] * self.layer_multipliers[index]
|
||||
)
|
||||
sizes = self.ngram_heads_vocab_sizes[start:end]
|
||||
offsets = self.ngram_heads_offsets[start:end]
|
||||
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
|
||||
id_blocks.append(ids[request_indices, adjusted_columns])
|
||||
return torch.cat(id_blocks, dim=-1)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
embedding = self.ngram_embedding
|
||||
if embedding.supports_prefetch:
|
||||
return embedding(hidden_states)
|
||||
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
|
||||
return self.ngram_embedding(ngram_ids).flatten(-2)
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> None:
|
||||
"""Start the pinned lookup while the preceding decoder layer runs."""
|
||||
embedding = self.ngram_embedding
|
||||
if not embedding.supports_prefetch:
|
||||
return
|
||||
ngram_ids = self.compute_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
embedding.start_prefetch(hidden_states, ngram_ids)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load hash buffers and checkpoint-split embedding rows."""
|
||||
|
||||
persistent_buffers = {
|
||||
"layer_multipliers": self.layer_multipliers,
|
||||
"ngram_heads_offsets": self.ngram_heads_offsets,
|
||||
"ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes,
|
||||
}
|
||||
loaded: set[str] = set()
|
||||
regular_weights: list[tuple[str, torch.Tensor]] = []
|
||||
shard_prefix = "ngram_embedding.shard_"
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
leaf_name = name.rsplit(".", 1)[-1]
|
||||
if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup":
|
||||
continue
|
||||
if name in persistent_buffers:
|
||||
buffer = persistent_buffers[name]
|
||||
if buffer.shape != loaded_weight.shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch for {name}: expected "
|
||||
f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}"
|
||||
)
|
||||
buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype))
|
||||
loaded.add(name)
|
||||
continue
|
||||
if name.startswith(shard_prefix) and name.endswith(".weight"):
|
||||
shard_text = name[len(shard_prefix) : -len(".weight")]
|
||||
if not shard_text.isdigit():
|
||||
regular_weights.append((name, loaded_weight))
|
||||
continue
|
||||
shard_index = int(shard_text)
|
||||
if shard_index >= self.split_ngram_parts:
|
||||
raise ValueError(
|
||||
f"PLE embedding shard index {shard_index} exceeds "
|
||||
f"split_ngram_parts={self.split_ngram_parts}"
|
||||
)
|
||||
embedding = self.ngram_embedding
|
||||
shard_size = (
|
||||
embedding.org_vocab_size + self.split_ngram_parts - 1
|
||||
) // self.split_ngram_parts
|
||||
checkpoint_start = shard_index * shard_size
|
||||
expected_rows = max(
|
||||
0,
|
||||
min(shard_size, embedding.org_vocab_size - checkpoint_start),
|
||||
)
|
||||
expected_shape = (expected_rows, embedding.embedding_dim)
|
||||
if tuple(loaded_weight.shape) != expected_shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch for PLE embedding shard {shard_index}: "
|
||||
f"expected {expected_shape}, got "
|
||||
f"{tuple(loaded_weight.shape)}"
|
||||
)
|
||||
embedding.weight.weight_loader(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=checkpoint_start,
|
||||
)
|
||||
loaded.add("ngram_embedding.weight")
|
||||
continue
|
||||
regular_weights.append((name, loaded_weight))
|
||||
|
||||
if regular_weights:
|
||||
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
|
||||
return loaded
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpNGramEmbedding",
|
||||
]
|
||||
@@ -2,10 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU-resident Qwen4Exp position-learning enhancement layers."""
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
@@ -18,24 +17,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
create_fp8_weight_parameter,
|
||||
is_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.model_executor.parameter import PerTensorScaleParameter
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
@@ -45,8 +26,8 @@ from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
)
|
||||
|
||||
from ..common.ple import PLEVocabParallelEmbedding
|
||||
from .ops.ple import ple_conv, ple_gate, ple_ngram_ids
|
||||
from .ngram_embedding import Qwen4ExpNGramEmbedding
|
||||
from .ops.ple import ple_conv, ple_gate
|
||||
|
||||
|
||||
class Qwen4ExpPLEGroupedNorm(nn.Module):
|
||||
@@ -82,437 +63,6 @@ class Qwen4ExpPLEGroupedNorm(nn.Module):
|
||||
return (normalized * (1.0 + self.weight.float())).to(input_dtype)
|
||||
|
||||
|
||||
class Qwen4ExpPLEFp8EmbeddingMethod(QuantizeMethodBase):
|
||||
"""FP8 PLE embedding with one global checkpoint scale."""
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size, params_dtype
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
weight = create_fp8_weight_parameter(
|
||||
sum(output_partition_sizes), input_size_per_partition, weight_loader
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
weight_scale = create_fp8_scale_parameter(
|
||||
PerTensorScaleParameter,
|
||||
output_partition_sizes,
|
||||
input_size_per_partition,
|
||||
None,
|
||||
weight_loader,
|
||||
scale_dtype=torch.float32,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: nn.Module) -> None:
|
||||
"""Reject FP8 PLE checkpoints without a global scale."""
|
||||
sentinel = torch.finfo(torch.float32).min
|
||||
if torch.any(layer.weight_scale == sentinel):
|
||||
raise ValueError("FP8 PLE checkpoint is missing its global scale")
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError("PLE FP8 weights only support embedding lookup")
|
||||
|
||||
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
|
||||
return F.embedding(input_, layer.weight)
|
||||
|
||||
|
||||
def _get_ple_embedding_quant_method(
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Select global-scale FP8 only for quantized PLE checkpoint shards."""
|
||||
|
||||
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
|
||||
if quant_config._resolve_quant_algo(prefix) == "FP8":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
return None
|
||||
|
||||
if not isinstance(quant_config, Fp8Config):
|
||||
return None
|
||||
if not quant_config.is_checkpoint_fp8_serialized:
|
||||
return None
|
||||
|
||||
ignored_layers = quant_config.ignored_layers
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
ignored_layers,
|
||||
quant_config.packed_modules_mapping,
|
||||
match_mode=quant_config.ignored_layers_match_mode,
|
||||
):
|
||||
return None
|
||||
# PLE checkpoint shards form one runtime embedding parameter.
|
||||
shard_prefix = f"{prefix}.shard_"
|
||||
if any(name.startswith(shard_prefix) for name in ignored_layers):
|
||||
return None
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
|
||||
|
||||
class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
_MASK64 = (1 << 64) - 1
|
||||
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
|
||||
_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
|
||||
_SPLITMIX_M2 = 0x94D049BB133111EB
|
||||
_PLE_LAYER_PRIME = 10007
|
||||
|
||||
@classmethod
|
||||
def _splitmix64(cls, value: int) -> int:
|
||||
"""Mix an integer into a deterministic unsigned 64-bit value."""
|
||||
value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64
|
||||
value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64
|
||||
value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64
|
||||
return (value ^ (value >> 31)) & cls._MASK64
|
||||
|
||||
@staticmethod
|
||||
def _is_prime_64(value: int) -> bool:
|
||||
"""Return whether a 64-bit integer is prime."""
|
||||
if value < 2:
|
||||
return False
|
||||
for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
|
||||
if value % prime == 0:
|
||||
return value == prime
|
||||
exponent = value - 1
|
||||
shifts = 0
|
||||
while exponent % 2 == 0:
|
||||
exponent //= 2
|
||||
shifts += 1
|
||||
for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022):
|
||||
if base % value == 0:
|
||||
continue
|
||||
witness = pow(base, exponent, value)
|
||||
if witness in (1, value - 1):
|
||||
continue
|
||||
for _ in range(shifts - 1):
|
||||
witness = pow(witness, 2, value)
|
||||
if witness == value - 1:
|
||||
break
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _nth_prime_after(cls, start: int, count: int) -> int:
|
||||
"""Return the ``count``-th prime strictly greater than ``start``."""
|
||||
prime = int(start)
|
||||
for _ in range(count):
|
||||
candidate = prime + 1
|
||||
if candidate <= 2:
|
||||
prime = 2
|
||||
continue
|
||||
if candidate % 2 == 0:
|
||||
candidate += 1
|
||||
while not cls._is_prime_64(candidate):
|
||||
candidate += 2
|
||||
prime = candidate
|
||||
return prime
|
||||
|
||||
@classmethod
|
||||
def _make_layer_multipliers(
|
||||
cls,
|
||||
*,
|
||||
ngram_size: int,
|
||||
unigram_vocab_size: int,
|
||||
seed: int,
|
||||
ple_dense_layer_id: int,
|
||||
) -> list[int]:
|
||||
"""Build deterministic hash multipliers for one PLE layer."""
|
||||
max_multiplier = ((1 << 63) - 1) // unigram_vocab_size
|
||||
half_bound = max(1, max_multiplier // 2)
|
||||
base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id
|
||||
multipliers = []
|
||||
for index in range(ngram_size):
|
||||
value = base_seed + cls._SPLITMIX_GAMMA * (index + 1)
|
||||
multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1)
|
||||
return multipliers
|
||||
|
||||
@classmethod
|
||||
def _make_vocab_layout(
|
||||
cls,
|
||||
*,
|
||||
ngram_vocab_size_base: int,
|
||||
ngram_heads: int,
|
||||
ple_dense_layer_id: int,
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
"""Build per-head vocabulary sizes, offsets, and total row count."""
|
||||
sizes: list[int] = []
|
||||
offsets: list[int] = []
|
||||
offset = 0
|
||||
for local_head in range(ngram_heads):
|
||||
global_head = ple_dense_layer_id * ngram_heads + local_head
|
||||
size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
|
||||
sizes.append(size)
|
||||
offsets.append(offset)
|
||||
offset += size
|
||||
return sizes, offsets, offset
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.embedding_dim = embedding_dim
|
||||
self.ngram_size = int(config.ngram_size)
|
||||
self.heads_per_ngram = int(config.heads_per_ngram)
|
||||
self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
|
||||
if self.ngram_size < 2:
|
||||
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
|
||||
if self.heads_per_ngram <= 0:
|
||||
raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}")
|
||||
if embedding_dim % self.ngram_heads:
|
||||
raise ValueError(
|
||||
"ple_embed_dim must be divisible by total ngram heads: "
|
||||
f"{embedding_dim} % {self.ngram_heads} != 0"
|
||||
)
|
||||
self.head_dim = embedding_dim // self.ngram_heads
|
||||
self.eos_token_id = int(config.eos_token_id)
|
||||
self.unigram_vocab_size = int(config.vocab_size)
|
||||
self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512))
|
||||
if self.split_ngram_parts <= 0:
|
||||
raise ValueError("split_ngram_parts must be positive")
|
||||
|
||||
multipliers = self._make_layer_multipliers(
|
||||
ngram_size=self.ngram_size,
|
||||
unigram_vocab_size=self.unigram_vocab_size,
|
||||
seed=int(getattr(config, "seed", 1234)),
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
)
|
||||
self.register_buffer(
|
||||
"layer_multipliers",
|
||||
torch.tensor(multipliers, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
sizes, offsets, total_vocab_size = self._make_vocab_layout(
|
||||
ngram_vocab_size_base=int(config.ngram_vocab_size_base),
|
||||
ngram_heads=self.ngram_heads,
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
)
|
||||
self.register_buffer(
|
||||
"ngram_heads_vocab_sizes",
|
||||
torch.tensor(sizes, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
self.register_buffer(
|
||||
"ngram_heads_offsets",
|
||||
torch.tensor(offsets, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
divisor = int(config.make_ngram_vocab_size_divisible_by)
|
||||
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
|
||||
self.ngram_embedding = PLEVocabParallelEmbedding(
|
||||
padded_vocab_size,
|
||||
self.head_dim,
|
||||
params_dtype=params_dtype,
|
||||
padding_size=divisor,
|
||||
prefix=f"{prefix}.ngram_embedding",
|
||||
quant_method=_get_ple_embedding_quant_method(
|
||||
quant_config, f"{prefix}.ngram_embedding"
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _shift_precompute(
|
||||
tokens: torch.Tensor, eos_token_id: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if tokens.dim() != 2:
|
||||
raise ValueError("tokens must be a 2D tensor")
|
||||
batch_size, seq_len = tokens.shape
|
||||
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
|
||||
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
|
||||
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
|
||||
previous_eos = torch.cat(
|
||||
[
|
||||
eos_positions.new_full((batch_size, 1), -1),
|
||||
previous_eos_inclusive[:, :-1],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
return positions, positions.unsqueeze(0) - previous_eos - 1
|
||||
|
||||
@staticmethod
|
||||
def _shift_apply(
|
||||
tokens: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
position_in_segment: torch.Tensor,
|
||||
shift: int,
|
||||
eos_token_id: int,
|
||||
) -> torch.Tensor:
|
||||
if shift == 0:
|
||||
return tokens
|
||||
source = positions - shift
|
||||
gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1)
|
||||
shifted = tokens.gather(1, gather_indices)
|
||||
valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift)
|
||||
return torch.where(valid, shifted, tokens.new_full((), eos_token_id))
|
||||
|
||||
def compute_ngram_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute n-gram embedding indices for the current request layout."""
|
||||
input_ids = input_ids.reshape(-1)
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
num_tokens = input_ids.shape[0]
|
||||
|
||||
if input_ids.is_cuda:
|
||||
return ple_ngram_ids(
|
||||
input_ids=input_ids,
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=ngram_context,
|
||||
layer_multipliers=self.layer_multipliers,
|
||||
ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes,
|
||||
ngram_heads_offsets=self.ngram_heads_offsets,
|
||||
eos_token_id=self.eos_token_id,
|
||||
heads_per_ngram=self.heads_per_ngram,
|
||||
output=output,
|
||||
)
|
||||
input_ids = input_ids.long()
|
||||
query_start_loc = query_start_loc.long()
|
||||
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
|
||||
packed = torch.full(
|
||||
(num_reqs, num_tokens),
|
||||
self.eos_token_id,
|
||||
device=input_ids.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
|
||||
request_indices.clamp_(max=num_reqs - 1)
|
||||
columns = (positions - query_start_loc[request_indices]).clamp(
|
||||
0, packed.shape[1] - 1
|
||||
)
|
||||
packed[request_indices, columns] = input_ids
|
||||
ngram_context = ngram_context[:num_reqs].to(
|
||||
device=input_ids.device, dtype=torch.long
|
||||
)
|
||||
|
||||
context = torch.cat([ngram_context, packed], dim=-1)
|
||||
positions_2d, position_in_segment = self._shift_precompute(
|
||||
context, self.eos_token_id
|
||||
)
|
||||
shifted = [context]
|
||||
for shift in range(1, self.ngram_size):
|
||||
shifted.append(
|
||||
self._shift_apply(
|
||||
context,
|
||||
positions_2d,
|
||||
position_in_segment,
|
||||
shift,
|
||||
self.eos_token_id,
|
||||
)
|
||||
)
|
||||
adjusted_columns = columns + self.ngram_size - 1
|
||||
id_blocks = []
|
||||
for ngram in range(2, self.ngram_size + 1):
|
||||
start = (ngram - 2) * self.heads_per_ngram
|
||||
end = start + self.heads_per_ngram
|
||||
mixed = shifted[0] * self.layer_multipliers[0]
|
||||
for index in range(1, ngram):
|
||||
mixed = torch.bitwise_xor(
|
||||
mixed, shifted[index] * self.layer_multipliers[index]
|
||||
)
|
||||
sizes = self.ngram_heads_vocab_sizes[start:end]
|
||||
offsets = self.ngram_heads_offsets[start:end]
|
||||
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
|
||||
id_blocks.append(ids[request_indices, adjusted_columns])
|
||||
return torch.cat(id_blocks, dim=-1)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
|
||||
return self.ngram_embedding(ngram_ids).flatten(-2)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load hash buffers and checkpoint-split embedding rows."""
|
||||
|
||||
persistent_buffers = {
|
||||
"layer_multipliers": self.layer_multipliers,
|
||||
"ngram_heads_offsets": self.ngram_heads_offsets,
|
||||
"ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes,
|
||||
}
|
||||
loaded: set[str] = set()
|
||||
regular_weights: list[tuple[str, torch.Tensor]] = []
|
||||
shard_prefix = "ngram_embedding.shard_"
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
leaf_name = name.rsplit(".", 1)[-1]
|
||||
if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup":
|
||||
continue
|
||||
if name in persistent_buffers:
|
||||
buffer = persistent_buffers[name]
|
||||
if buffer.shape != loaded_weight.shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch for {name}: expected "
|
||||
f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}"
|
||||
)
|
||||
buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype))
|
||||
loaded.add(name)
|
||||
continue
|
||||
if name.startswith(shard_prefix) and name.endswith(".weight"):
|
||||
shard_text = name[len(shard_prefix) : -len(".weight")]
|
||||
if not shard_text.isdigit():
|
||||
regular_weights.append((name, loaded_weight))
|
||||
continue
|
||||
shard_index = int(shard_text)
|
||||
if shard_index >= self.split_ngram_parts:
|
||||
raise ValueError(
|
||||
f"PLE embedding shard index {shard_index} exceeds "
|
||||
f"split_ngram_parts={self.split_ngram_parts}"
|
||||
)
|
||||
embedding = self.ngram_embedding
|
||||
shard_size = (
|
||||
embedding.org_vocab_size + self.split_ngram_parts - 1
|
||||
) // self.split_ngram_parts
|
||||
checkpoint_start = shard_index * shard_size
|
||||
expected_rows = max(
|
||||
0,
|
||||
min(shard_size, embedding.org_vocab_size - checkpoint_start),
|
||||
)
|
||||
expected_shape = (expected_rows, embedding.embedding_dim)
|
||||
if tuple(loaded_weight.shape) != expected_shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch for PLE embedding shard {shard_index}: "
|
||||
f"expected {expected_shape}, got "
|
||||
f"{tuple(loaded_weight.shape)}"
|
||||
)
|
||||
embedding.weight.weight_loader(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=checkpoint_start,
|
||||
)
|
||||
loaded.add("ngram_embedding.weight")
|
||||
continue
|
||||
regular_weights.append((name, loaded_weight))
|
||||
|
||||
if regular_weights:
|
||||
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
|
||||
return loaded
|
||||
|
||||
|
||||
class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -547,6 +97,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
@@ -588,10 +140,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def _get_embedding_weight_scale(self) -> torch.Tensor | None:
|
||||
embedding = getattr(self.ple_embedding, "ngram_embedding", None)
|
||||
return getattr(embedding, "weight_scale", None)
|
||||
|
||||
def _dequantize_embeddings(
|
||||
self,
|
||||
embeddings: torch.Tensor,
|
||||
@@ -599,14 +147,25 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize PLE lookup output."""
|
||||
|
||||
if not is_fp8(embeddings):
|
||||
return embeddings
|
||||
weight_scale = self._get_embedding_weight_scale()
|
||||
if weight_scale is None:
|
||||
raise RuntimeError("FP8 PLE embedding is missing its global scale")
|
||||
if weight_scale.device != embeddings.device:
|
||||
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
|
||||
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
|
||||
return self.ple_embedding.ngram_embedding.dequantize(
|
||||
embeddings,
|
||||
output_dtype,
|
||||
)
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> None:
|
||||
"""Start the pinned PLE lookup while the preceding decoder layer runs."""
|
||||
self.ple_embedding.start_prefetch(
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
@@ -827,7 +386,12 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
f"token length, got {input_ids.shape[0]} and "
|
||||
f"{hidden_states.shape[0]}"
|
||||
)
|
||||
embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context)
|
||||
embeddings = self.ple_embedding(
|
||||
hidden_states,
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
)
|
||||
embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype)
|
||||
kv, _ = self.kv_proj(embeddings)
|
||||
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
|
||||
@@ -845,7 +409,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpNGramEmbedding",
|
||||
"Qwen4ExpPLEGroupedNorm",
|
||||
"Qwen4ExpPLELayer",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user