diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index e0bb67b1bb..ecaf287956 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -63,3 +63,31 @@ steps: commands: - pip install "vllm-gguf-plugin >= 0.0.2" - pytest -v -s plugins_tests/gguf + +- label: BitsAndBytes Plugin + key: bitsandbytes-plugin + device: h200_18gb + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + soft_fail: true + optional: true + source_file_dependencies: + - vllm/model_executor/layers/quantization + - tests/plugins_tests/bitsandbytes + commands: + - pip install "vllm-bnb-plugin >= 0.0.1" + - pytest -v -s plugins_tests/bitsandbytes -m 'not distributed' + +- label: BitsAndBytes Plugin (2 GPUs) + key: bitsandbytes-plugin-2-gpus + timeout_in_minutes: 15 + working_dir: "/vllm-workspace/tests" + num_devices: 2 + soft_fail: true + optional: true + source_file_dependencies: + - vllm/model_executor/layers/quantization + - tests/plugins_tests/bitsandbytes + commands: + - pip install "vllm-bnb-plugin >= 0.0.1" + - pytest -v -s plugins_tests/bitsandbytes -m 'distributed(num_gpus=2)' diff --git a/docker/Dockerfile b/docker/Dockerfile index c020488828..22af82b0b6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -779,20 +779,12 @@ RUN set -eux; \ # Install vllm-openai dependencies (saves ~2.6s per build) # These are stable packages that don't depend on vLLM itself -# From versions.json: .bitsandbytes.x86_64, .bitsandbytes.arm64 # From versions.json: .openai_server_extras.timm, .openai_server_extras.runai_model_streamer -ARG BITSANDBYTES_VERSION_X86=0.46.1 -ARG BITSANDBYTES_VERSION_ARM64=0.42.0 ARG TIMM_VERSION=">=1.0.17" ARG RUNAI_MODEL_STREAMER_VERSION=">=0.15.7" RUN --mount=type=cache,target=/opt/uv/cache \ - if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ - BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_ARM64}"; \ - else \ - BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_X86}"; \ - fi; \ uv pip install --system accelerate 'modelscope<1.38' \ - "bitsandbytes>=${BITSANDBYTES_VERSION}" "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs,azure]${RUNAI_MODEL_STREAMER_VERSION}" + "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs,azure]${RUNAI_MODEL_STREAMER_VERSION}" # ============================================================ # VLLM INSTALLATION (depends on build stage) diff --git a/docker/versions.json b/docker/versions.json index 185af3102b..68db26e500 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -76,12 +76,6 @@ "GDRCOPY_OS_VERSION": { "default": "Ubuntu24_04" }, - "BITSANDBYTES_VERSION_X86": { - "default": "0.46.1" - }, - "BITSANDBYTES_VERSION_ARM64": { - "default": "0.42.0" - }, "TIMM_VERSION": { "default": ">=1.0.17" }, diff --git a/docs/features/quantization/bnb.md b/docs/features/quantization/bnb.md index 53419e0672..9a27b509bd 100644 --- a/docs/features/quantization/bnb.md +++ b/docs/features/quantization/bnb.md @@ -4,10 +4,12 @@ vLLM now supports [BitsAndBytes](https://github.com/TimDettmers/bitsandbytes) fo BitsAndBytes quantizes models to reduce memory usage and enhance performance without significantly sacrificing accuracy. Compared to other quantization methods, BitsAndBytes eliminates the need for calibrating the quantized model with input data. -Below are the steps to utilize BitsAndBytes with vLLM. +BitsAndBytes support is provided by the out-of-tree +[`vllm-bnb-plugin`](https://github.com/vllm-project/vllm-bnb-plugin). +Install the plugin before using BitsAndBytes with vLLM: ```bash -pip install bitsandbytes>=0.49.2 +uv pip install vllm-bnb-plugin ``` vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint. diff --git a/examples/features/lora/lora_with_quantization_offline.py b/examples/features/lora/lora_with_quantization_offline.py index ee5bbd82cd..ef6dbd4534 100644 --- a/examples/features/lora/lora_with_quantization_offline.py +++ b/examples/features/lora/lora_with_quantization_offline.py @@ -5,6 +5,7 @@ This example shows how to use LoRA with different quantization techniques for offline inference. Requires HuggingFace credentials for access. +The bitsandbytes example also requires installing vllm-bnb-plugin. """ import gc diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 08e7237951..b330f15f27 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -70,8 +70,6 @@ azure-storage-blob==12.28.0 # via runai-model-streamer-azure backoff==2.2.1 # via -r requirements/test/cuda.in -bitsandbytes==0.49.2 - # via -r requirements/test/cuda.in black==24.10.0 # via datamodel-code-generator blake3==1.0.9 @@ -535,7 +533,6 @@ numpy==2.2.6 # -r requirements/test/../common.txt # accelerate # albumentations - # bitsandbytes # bm25s # contourpy # cupy-cuda12x @@ -642,7 +639,6 @@ outlines-core==0.2.14 packaging==24.2 # via # accelerate - # bitsandbytes # black # datamodel-code-generator # datasets @@ -1126,7 +1122,6 @@ torch==2.13.0+cpu # via # -r requirements/test/cuda.in # accelerate - # bitsandbytes # compressed-tensors # encodec # instanttensor diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 116f405444..c3d3e1289a 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -43,7 +43,6 @@ transformers==5.14.1 tokenizers==0.22.2 schemathesis>=4.0.0 # Required for openai schema test. # quantization -bitsandbytes==0.49.2 buildkite-test-collector==0.1.9 torch-abi-audit # CI check for PyTorch stable ABI compliance diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index d722581275..aef8c1aa09 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -75,8 +75,6 @@ azure-storage-blob==12.28.0 # via runai-model-streamer-azure backoff==2.2.1 # via -r requirements/test/cuda.in -bitsandbytes==0.49.2 - # via -r requirements/test/cuda.in black==24.10.0 # via datamodel-code-generator blake3==1.0.9 @@ -566,7 +564,6 @@ numpy==2.2.6 # -r requirements/test/../common.txt # accelerate # albumentations - # bitsandbytes # bm25s # contourpy # cupy-cuda12x @@ -726,7 +723,6 @@ outlines-core==0.2.14 packaging==24.2 # via # accelerate - # bitsandbytes # black # datamodel-code-generator # datasets @@ -1225,7 +1221,6 @@ torch==2.13.0+cu130 # -c requirements/cuda.txt # -r requirements/test/cuda.in # accelerate - # bitsandbytes # compressed-tensors # encodec # instanttensor diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index a98dca7394..ac38f761cf 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -33,7 +33,6 @@ transformers==5.14.1 tokenizers==0.22.2 schemathesis>=4.0.0 # Required for openai schema test. # quantization -bitsandbytes>=0.49.2 buildkite-test-collector==0.1.9 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index f8a0ece15b..eb93b0c24c 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -39,7 +39,6 @@ transformers==5.14.1 tokenizers==0.22.2 schemathesis>=4.0.0 # Required for openai schema test # quantization -bitsandbytes==0.49.2 buildkite-test-collector==0.1.9 tilelang==0.1.10 diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 9b00aee54d..c5fe450568 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -72,8 +72,6 @@ azure-storage-blob==12.28.0 # via runai-model-streamer-azure backoff==2.2.1 # via -r requirements/test/rocm.in -bitsandbytes==0.49.2 - # via -r requirements/test/rocm.in black==26.3.1 # via datamodel-code-generator blake3==1.0.8 @@ -568,7 +566,6 @@ numpy==2.2.6 # -r requirements/test/../common.txt # accelerate # albumentations - # bitsandbytes # bm25s # contourpy # cupy-cuda12x @@ -698,7 +695,6 @@ packaging==26.0 # via # -c requirements/rocm.txt # accelerate - # bitsandbytes # black # datasets # evaluate diff --git a/tests/conftest.py b/tests/conftest.py index f9003e55e7..e277b35dd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -521,10 +521,7 @@ class HfRunner: ): model = model.to(dtype=self.dtype) - if ( - getattr(model, "quantization_method", None) != "bitsandbytes" - and len({p.device for p in model.parameters()}) < 2 - ): + if len({p.device for p in model.parameters()}) < 2: model = model.to(device=self.device) self.model = model diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index d2e2d08dd9..aa283419bf 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -343,40 +343,6 @@ class TestWeightLoadingWithPaddedHiddenSize: torch.zeros(original_hidden, padded_intermediate - original_intermediate), ) - def test_bnb_shape_mismatch_raises(self): - """BnB + padded hidden_size should raise via weight_loader.""" - from unittest.mock import MagicMock - - num_experts = 1 - padded_packed = 3072 # padded packed size - original_packed = 2688 # original packed size - - # Build a param that looks like a BnB 4-bit MoE weight. - param_data = torch.zeros(num_experts, padded_packed, 1, dtype=torch.uint8) - param = torch.nn.Parameter(param_data, requires_grad=False) - param.use_bitsandbytes_4bit = True - - loaded_weight = torch.randint(0, 255, (original_packed, 1), dtype=torch.uint8) - - # Minimal RoutedExperts mock so weight_loader reaches the BnB path. - moe = MagicMock(spec=RoutedExperts) - moe.quant_config = None - moe.quant_method = MagicMock() - moe.quant_method.__class__.__name__ = "BitsAndBytesMethod" - moe._expert_map = None - moe.tp_rank = 0 - - # Call the real weight_loader (unbound) with our mock as self. - with pytest.raises(ValueError, match="BitsAndBytes"): - RoutedExperts.weight_loader( - moe, - param, - loaded_weight, - weight_name="w2", - shard_id="w2", - expert_id=0, - ) - class TestLoadWeightsExpertBias: """Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index 323ec760bf..ed02f2934e 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -189,13 +189,6 @@ def test_distributed( [ ("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {}), ("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {}), - ( - "meta-llama/Llama-3.2-1B-Instruct", - { - "quantization": "bitsandbytes", - }, - ), - ("unsloth/tinyllama-bnb-4bit", {}), ], ) @pytest.mark.parametrize("max_tokens", [32]) diff --git a/tests/plugins_tests/bitsandbytes/__init__.py b/tests/plugins_tests/bitsandbytes/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/plugins_tests/bitsandbytes/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/models/quantization/test_bitsandbytes.py b/tests/plugins_tests/bitsandbytes/test_bitsandbytes.py similarity index 72% rename from tests/models/quantization/test_bitsandbytes.py rename to tests/plugins_tests/bitsandbytes/test_bitsandbytes.py index c2f1a89df2..1e96598d4d 100644 --- a/tests/models/quantization/test_bitsandbytes.py +++ b/tests/plugins_tests/bitsandbytes/test_bitsandbytes.py @@ -1,9 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests whether bitsandbytes computation is enabled correctly. - -Run `pytest tests/quantization/test_bitsandbytes.py`. -""" +"""End-to-end tests for the out-of-tree bitsandbytes plugin.""" import types from unittest.mock import MagicMock, patch @@ -12,13 +9,12 @@ import pytest from packaging.version import Version from transformers import BitsAndBytesConfig from transformers import __version__ as TRANSFORMERS_VERSION +from vllm_bnb_plugin import bitsandbytes_loader as bnb -from tests.quantization.utils import is_quant_method_supported -from vllm.model_executor.model_loader import bitsandbytes_loader as bnb from vllm.platforms import current_platform -from ...utils import compare_two_settings, multi_gpu_test -from ..utils import check_embeddings_close, check_logprobs_close +from ...models.utils import check_embeddings_close, check_logprobs_close +from ...utils import multi_gpu_test if current_platform.is_rocm(): from vllm.platforms.rocm import on_cdna @@ -44,7 +40,7 @@ models_4bit_to_moe_test = [ ("allenai/OLMoE-1B-7B-0125-Instruct", "quantize moe model inflight"), ] -models_pre_qaunt_4bit_to_test = [ +models_pre_quant_4bit_to_test = [ ( "PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed", "read pre-quantized 4-bit FP4 model", @@ -58,10 +54,64 @@ models_pre_quant_8bit_to_test = [ ] -@pytest.mark.skipif( - not is_quant_method_supported("bitsandbytes"), - reason="bitsandbytes is not supported on this GPU type.", -) +def log_generated_texts(prompts, outputs, runner_name): + logged_texts = [] + for i, (_, generated_text) in enumerate(outputs): + logged_texts.append( + { + "prompt": prompts[i], + "runner_name": runner_name, + "generated_text": generated_text, + } + ) + return logged_texts + + +def validate_generated_texts( + hf_runner, + vllm_runner, + prompts, + model_name, + pre_quant=False, + hf_model_kwargs=None, + vllm_tp_size=1, + max_tokens=8, +): + with vllm_runner( + model_name, + quantization=None if pre_quant else "bitsandbytes", + tensor_parallel_size=vllm_tp_size, + enforce_eager=False, + default_torch_num_threads=1, + tokenizer_mode="hf", + load_format="hf", + config_format="hf", + ) as llm: + vllm_outputs = llm.generate_greedy(prompts, max_tokens) + vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner") + + if hf_model_kwargs is None: + hf_model_kwargs = {} + + with hf_runner( + model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1 + ) as llm: + hf_outputs = llm.generate_greedy(prompts, max_tokens) + hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner") + + for hf_log, vllm_log in zip(hf_logs, vllm_logs): + hf_str = hf_log["generated_text"] + vllm_str = vllm_log["generated_text"] + prompt = hf_log["prompt"] + assert hf_str == vllm_str, ( + f"Model: {model_name}" + f"Mismatch between HF and vLLM outputs:\n" + f"Prompt: {prompt}\n" + f"HF Output: '{hf_str}'\n" + f"vLLM Output: '{vllm_str}'" + ) + + @pytest.mark.parametrize("model_name, description", models_4bit_to_test) def test_load_4bit_bnb_model( hf_runner, vllm_runner, example_prompts, model_name, description @@ -72,11 +122,7 @@ def test_load_4bit_bnb_model( ) -@pytest.mark.skipif( - not is_quant_method_supported("bitsandbytes"), - reason="bitsandbytes is not supported on this GPU type.", -) -@pytest.mark.parametrize("model_name, description", models_pre_qaunt_4bit_to_test) +@pytest.mark.parametrize("model_name, description", models_pre_quant_4bit_to_test) def test_load_pre_quant_4bit_bnb_model( hf_runner, vllm_runner, example_prompts, model_name, description ) -> None: @@ -85,10 +131,6 @@ def test_load_pre_quant_4bit_bnb_model( ) -@pytest.mark.skipif( - not is_quant_method_supported("bitsandbytes"), - reason="bitsandbytes is not supported on this GPU type.", -) @pytest.mark.parametrize("model_name, description", models_pre_quant_8bit_to_test) def test_load_8bit_bnb_model( hf_runner, vllm_runner, example_prompts, model_name, description @@ -98,10 +140,6 @@ def test_load_8bit_bnb_model( ) -@pytest.mark.skipif( - not is_quant_method_supported("bitsandbytes"), - reason="bitsandbytes is not supported on this GPU type.", -) @pytest.mark.parametrize("model_name, description", models_4bit_to_test) @multi_gpu_test(num_gpus=2) def test_load_tp_4bit_bnb_model( @@ -119,44 +157,28 @@ def test_load_tp_4bit_bnb_model( ) -@pytest.mark.skipif( - not is_quant_method_supported("bitsandbytes"), - reason="bitsandbytes is not supported on this GPU type.", -) @pytest.mark.parametrize("model_name, description", models_4bit_to_test) @multi_gpu_test(num_gpus=2) -def test_load_pp_4bit_bnb_model(model_name, description) -> None: - common_args = [ - "--disable-log-stats", - "--dtype", - "bfloat16", - "--enable-prefix-caching", - "--quantization", - "bitsandbytes", - "--gpu-memory-utilization", - "0.7", - ] - pp_args = [ - *common_args, - "--pipeline-parallel-size", - "2", - ] - compare_two_settings( +def test_load_pp_4bit_bnb_model( + hf_runner, vllm_runner, example_prompts, model_name, description +) -> None: + hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True)) + validate_generated_texts( + hf_runner, + vllm_runner, + example_prompts[:1], model_name, - common_args, - pp_args, + False, + hf_model_kwargs, + vllm_tp_size=2, ) @pytest.mark.skipif( Version(TRANSFORMERS_VERSION) >= Version("5.0.0"), - reason="Need to add support for quantizing MoE experts with bnb" - " in transformers v5. See" - " https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1849", -) -@pytest.mark.skipif( - not is_quant_method_supported("bitsandbytes"), - reason="bitsandbytes is not supported on this GPU type.", + reason="Need to add support for quantizing MoE experts with bnb in " + "transformers v5. See https://github.com/bitsandbytes-foundation/" + "bitsandbytes/issues/1849", ) @pytest.mark.parametrize("model_name, description", models_4bit_to_moe_test) def test_4bit_bnb_moe_model( @@ -193,10 +215,6 @@ def test_4bit_bnb_moe_model( ) -@pytest.mark.skipif( - not is_quant_method_supported("bitsandbytes"), - reason="bitsandbytes is not supported on this GPU type.", -) @pytest.mark.parametrize("model_name, description", models_4bit_to_embedding_test) @pytest.mark.parametrize("dtype", ["half"]) def test_4bit_bnb_embedding_model( @@ -207,15 +225,8 @@ def test_4bit_bnb_embedding_model( example_prompts, dtype: str, ) -> None: - # The example_prompts has ending "\n", for example: - # "Write a short story about a robot that dreams for the first time.\n" - # sentence_transformers will strip the input texts, see: - # https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159 - # This makes the input_ids different between hf_model and vllm_model. - # So we need to strip the input texts to avoid test failing. example_prompts = [str(s).strip() for s in example_prompts] - # Inflight 4bit quantization with vllm_runner( model_name, runner="pooling", @@ -245,70 +256,7 @@ def test_4bit_bnb_embedding_model( ) -def log_generated_texts(prompts, outputs, runner_name): - logged_texts = [] - for i, (_, generated_text) in enumerate(outputs): - log_entry = { - "prompt": prompts[i], - "runner_name": runner_name, - "generated_text": generated_text, - } - logged_texts.append(log_entry) - return logged_texts - - -def validate_generated_texts( - hf_runner, - vllm_runner, - prompts, - model_name, - pre_quant=False, - hf_model_kwargs=None, - vllm_tp_size=1, - max_tokens=8, -): - # NOTE: run vLLM first, as it requires a clean process - # when using distributed inference - with vllm_runner( - model_name, - quantization=None if pre_quant else "bitsandbytes", - tensor_parallel_size=vllm_tp_size, - enforce_eager=False, - default_torch_num_threads=1, - tokenizer_mode="hf", - load_format="hf", - config_format="hf", - ) as llm: - vllm_outputs = llm.generate_greedy(prompts, max_tokens) - vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner") - - if hf_model_kwargs is None: - hf_model_kwargs = {} - - # Run with HF runner - with hf_runner( - model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1 - ) as llm: - hf_outputs = llm.generate_greedy(prompts, max_tokens) - hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner") - - # Compare the generated strings - for hf_log, vllm_log in zip(hf_logs, vllm_logs): - hf_str = hf_log["generated_text"] - vllm_str = vllm_log["generated_text"] - prompt = hf_log["prompt"] - assert hf_str == vllm_str, ( - f"Model: {model_name}" - f"Mismatch between HF and vLLM outputs:\n" - f"Prompt: {prompt}\n" - f"HF Output: '{hf_str}'\n" - f"vLLM Output: '{vllm_str}'" - ) - - def test_bitsandbytes_passes_revision_by_name(): - # revision must reach download_safetensors_index_file_from_hf as the - # ``revision`` keyword, not a positional slot. fake_self = types.SimpleNamespace( load_config=types.SimpleNamespace(download_dir="/cache"), _get_weight_files=MagicMock( diff --git a/tests/plugins_tests/bitsandbytes/test_transformers.py b/tests/plugins_tests/bitsandbytes/test_transformers.py new file mode 100644 index 0000000000..398d869ade --- /dev/null +++ b/tests/plugins_tests/bitsandbytes/test_transformers.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Transformers backend tests for bitsandbytes plugin models.""" + +import pytest + +from vllm.platforms import current_platform + +from ...models.utils import check_logprobs_close + +if current_platform.is_rocm(): + from vllm.platforms.rocm import on_cdna + + pytestmark = pytest.mark.skipif( + on_cdna(), + reason="bitsandbytes not supported on CDNA (warp size 64 limitation)", + ) + + +@pytest.mark.parametrize( + "model, quantization_kwargs", + [ + ( + "meta-llama/Llama-3.2-1B-Instruct", + { + "quantization": "bitsandbytes", + }, + ), + ("unsloth/tinyllama-bnb-4bit", {}), + ], +) +@pytest.mark.parametrize("max_tokens", [32]) +@pytest.mark.parametrize("num_logprobs", [5]) +def test_transformers_backend_quantization( + vllm_runner, + example_prompts, + model: str, + quantization_kwargs: dict[str, str], + max_tokens: int, + num_logprobs: int, +) -> None: + with vllm_runner( + model, + model_impl="auto", + enforce_eager=True, + **quantization_kwargs, # type: ignore[arg-type] + ) as vllm_model: + vllm_outputs = vllm_model.generate_greedy_logprobs( + example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs + ) + + with vllm_runner( + model, + model_impl="transformers", + enforce_eager=True, + **quantization_kwargs, # type: ignore[arg-type] + ) as vllm_model: + model_config = vllm_model.llm.llm_engine.model_config + assert model_config.using_transformers_backend() + + transformers_outputs = vllm_model.generate_greedy_logprobs( + example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs + ) + + check_logprobs_close( + outputs_0_lst=transformers_outputs, + outputs_1_lst=vllm_outputs, + name_0="transformers", + name_1="vllm", + ) diff --git a/vllm/config/load.py b/vllm/config/load.py index b21eab144a..e090214403 100644 --- a/vllm/config/load.py +++ b/vllm/config/load.py @@ -49,7 +49,6 @@ class LoadConfig: Streamer. - "runai_streamer_sharded" will load weights from pre-sharded checkpoint files using Run:ai Model Streamer. - - "bitsandbytes" will load the weights using bitsandbytes quantization. - "sharded_state" will load weights from pre-sharded checkpoint files, supporting efficient loading of tensor-parallel models. - "mistral" will load weights from consolidated safetensors files used by diff --git a/vllm/config/model.py b/vllm/config/model.py index b01e6c8c68..7d39ea0df7 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -849,7 +849,6 @@ class ModelConfig: self._try_verify_and_update_model_config() self._verify_quantization() self._verify_cuda_graph() - self._verify_bnb_config() def _supports_multimodal_for_mm_prefix(self) -> bool: """Whether multimodal inputs can still appear for this deployment. @@ -1317,34 +1316,6 @@ class ModelConfig: ) self.enforce_eager = True - def _verify_bnb_config(self) -> None: - """ - The current version of bitsandbytes (0.46.1) with 8-bit models does not - yet support CUDA graph. - # TODO Remove this when bitsandbytes supports. - """ - is_bitsandbytes = self.quantization == "bitsandbytes" - has_quantization_config = self.model_arch_config.quantization_config is not None - is_8bit = ( - self.model_arch_config.quantization_config.get("load_in_8bit", False) # type: ignore[union-attr] - if has_quantization_config - else False - ) - if all( - [ - is_bitsandbytes, - has_quantization_config, - is_8bit, - not self.enforce_eager, - ] - ): - logger.warning( - "CUDA graph is not supported on BitsAndBytes 8bit yet, " - "fallback to the eager mode." - ) - - self.enforce_eager = True - def _verify_with_expert_parallelism(self) -> None: if not self.is_moe: raise ValueError( diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index da4359f9ad..f84e3c355b 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -1794,9 +1794,6 @@ class EngineArgs: ) def create_load_config(self) -> LoadConfig: - if self.quantization == "bitsandbytes": - self.load_format = "bitsandbytes" - if self.load_format == "tensorizer": if hasattr(self.model_loader_extra_config, "to_serializable"): self.model_loader_extra_config = ( @@ -2360,10 +2357,6 @@ class EngineArgs: "decreasing num_speculative_tokens" ) - # bitsandbytes pre-quantized model need a specific model loader - if model_config.quantization == "bitsandbytes": - self.quantization = self.load_format = "bitsandbytes" - # Attention config overrides attention_config = copy.deepcopy(self.attention_config) if self.attention_backend is not None: diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 6c31f817a3..fa50456a61 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -674,44 +674,6 @@ class RoutedExperts(PluggableLayer): # based on the shard id. This will be whatever # dimension intermediate_size_per_partition is used. SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0} - - # Case for BitsAndBytes - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) - if use_bitsandbytes_4bit: - shard_dim = 0 - - expert_data = param.data[expert_id] - if shard_id == "w2": - # BnB params are stored as flat packed tensors (e.g. - # (packed_size, 1)), not in the logical weight layout. - # Narrowing packed data for hidden-dim padding is not - # meaningful, so require an exact shape match. - if expert_data.shape != loaded_weight.shape: - raise ValueError( - "BitsAndBytes quantization with padded hidden_size " - "(e.g., from DeepEP) is not supported. " - f"Parameter shape {tuple(expert_data.shape)} != " - f"checkpoint shape {tuple(loaded_weight.shape)}" - ) - expert_data.copy_(loaded_weight) - elif shard_id in ("w1", "w3"): - # BnB stores weights as flat packed tensors. _load_w13 is - # still used to split the w1/w3 portions along shard_dim. - # _narrow_expert_data_for_padding will be a no-op since - # packed sizes should already match; if DeepEP padding - # causes a mismatch the copy_() will fail with a clear - # shape error. - full_load = True - self._load_w13( - shard_id=shard_id, - shard_dim=shard_dim, - loaded_weight=loaded_weight, - expert_data=expert_data, - tp_rank=self.moe_config.tp_rank, - load_full=full_load, - ) - return True if return_success else None - shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id] if is_transposed: shard_dim = int(not shard_dim) diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index e4662148ff..050e7a0acd 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import itertools from abc import abstractmethod from collections.abc import Iterable from typing import Any @@ -96,23 +95,6 @@ def adjust_block_scale_shard( return shard_size, shard_offset -def adjust_bitsandbytes_4bit_shard( - param: Parameter, - shard_offsets: dict[str, tuple[int, int]], - loaded_shard_id: str, -) -> tuple[int, int]: - """Adjust the quantization offsets and sizes for BitsAndBytes sharding.""" - - total, _ = shard_offsets["total"] - orig_offset, orig_size = shard_offsets[loaded_shard_id] - - quantized_total = param.data.shape[0] - quantized_offset = orig_offset * quantized_total // total - quantized_size = orig_size * quantized_total // total - - return quantized_size, quantized_offset - - def adjust_scalar_to_fused_array( param_data: torch.Tensor, loaded_weight: torch.Tensor, @@ -561,10 +543,6 @@ class ColumnParallelLinear(LinearBase): output_dim = getattr(param, "output_dim", None) is_sharded_weight = getattr(param, "is_sharded_weight", False) - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) - # bitsandbytes loads the weights of the specific portion - # no need to narrow - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit param_data = param.data if output_dim is not None and not is_sharded_weight: @@ -775,16 +753,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear): else self.output_sizes ) current_shard_offset = 0 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) - if ( - use_bitsandbytes_4bit - and isinstance(loaded_shard_id, tuple) - and self.tp_size > 1 - ): - raise NotImplementedError( - "Shard id with multiple indices is not supported " - "for BNB quantization with TP yet." - ) shard_offsets: list[tuple[int, int, int]] = [] for i, output_size in enumerate(output_sizes): shard_offsets.append((i, current_shard_offset, output_size)) @@ -809,17 +777,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear): param, shard_size, shard_offset ) - if use_bitsandbytes_4bit: - index = list(itertools.accumulate([0] + self.output_sizes)) - orig_offsets = { - str(i): (index[i], size) - for i, size in enumerate(self.output_sizes) - } - orig_offsets["total"] = (self.output_size, 0) - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( - param, orig_offsets, str(shard_id) - ) - loaded_weight_shard = loaded_weight.narrow( output_dim, shard_offset, shard_size ) @@ -851,21 +808,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear): param, shard_size, shard_offset ) - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) is_sharded_weight = getattr(param, "is_sharded_weight", False) - # bitsandbytes loads the weights of the specific portion - # no need to narrow - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit - - if use_bitsandbytes_4bit: - index = list(itertools.accumulate([0] + self.output_sizes)) - orig_offsets = { - str(i): (index[i], size) for i, size in enumerate(self.output_sizes) - } - orig_offsets["total"] = (self.output_size, 0) - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( - param, orig_offsets, str(loaded_shard_id) - ) param_data = param_data.narrow(output_dim, shard_offset, shard_size) start_idx = self.tp_rank * shard_size if not is_sharded_weight: @@ -1259,8 +1202,6 @@ class QKVParallelLinear(ColumnParallelLinear): self.total_num_kv_heads * self.v_head_size, ), ] - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) - packed_dim = getattr(param, "packed_dim", None) for shard_id, shard_offset, shard_size in shard_offsets: # Special case for Quantized Weights. @@ -1282,30 +1223,6 @@ class QKVParallelLinear(ColumnParallelLinear): param, shard_size, shard_offset ) - if use_bitsandbytes_4bit: - orig_qkv_offsets = { - "q": (0, self.total_num_heads * self.head_size), - "k": ( - self.total_num_heads * self.head_size, - self.total_num_kv_heads * self.head_size, - ), - "v": ( - (self.total_num_heads + self.total_num_kv_heads) - * self.head_size, - self.total_num_kv_heads * self.v_head_size, - ), - "total": ( - (self.total_num_heads + self.total_num_kv_heads) - * self.head_size - + self.total_num_kv_heads * self.v_head_size, - 0, - ), - } - - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( - param, orig_qkv_offsets, shard_id - ) - loaded_weight_shard = loaded_weight.narrow( output_dim, shard_offset, shard_size ) @@ -1345,33 +1262,7 @@ class QKVParallelLinear(ColumnParallelLinear): param, shard_size, shard_offset ) - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) is_sharded_weight = getattr(param, "is_sharded_weight", False) - # bitsandbytes loads the weights of the specific portion - # no need to narrow - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit - - if use_bitsandbytes_4bit: - orig_qkv_offsets = { - "q": (0, self.num_heads * self.head_size), - "k": ( - self.num_heads * self.head_size, - self.num_kv_heads * self.head_size, - ), - "v": ( - (self.num_heads + self.num_kv_heads) * self.head_size, - self.num_kv_heads * self.v_head_size, - ), - "total": ( - (self.num_heads + self.num_kv_heads) * self.head_size - + self.num_kv_heads * self.v_head_size, - 0, - ), - } - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( - param, orig_qkv_offsets, loaded_shard_id - ) - param_data = param_data.narrow(output_dim, shard_offset, shard_size) if loaded_shard_id == "q": shard_rank = self.tp_rank @@ -1716,11 +1607,7 @@ class RowParallelLinear(LinearBase): def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): input_dim = getattr(param, "input_dim", None) - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) is_sharded_weight = getattr(param, "is_sharded_weight", False) - # bitsandbytes loads the weights of the specific portion - # no need to narrow - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit param_data = param.data if input_dim is not None and not is_sharded_weight: diff --git a/vllm/model_executor/layers/quantization/__init__.py b/vllm/model_executor/layers/quantization/__init__.py index 55e815b2a9..28dc9a688e 100644 --- a/vllm/model_executor/layers/quantization/__init__.py +++ b/vllm/model_executor/layers/quantization/__init__.py @@ -25,7 +25,6 @@ QuantizationMethods = Literal[ "awq_marlin", "humming", "compressed-tensors", - "bitsandbytes", "experts_int8", "quark", "moe_wna16", @@ -117,7 +116,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: from .auto_awq import AutoAWQConfig from .auto_gptq import AutoGPTQConfig - from .bitsandbytes import BitsAndBytesConfig from .compressed_tensors.compressed_tensors import ( CompressedTensorsConfig, ) @@ -153,7 +151,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: "gptq": AutoGPTQConfig, "gptq_marlin": AutoGPTQConfig, "compressed-tensors": CompressedTensorsConfig, - "bitsandbytes": BitsAndBytesConfig, "experts_int8": ExpertsInt8Config, "quark": QuarkConfig, "moe_wna16": MoeWNA16Config, diff --git a/vllm/model_executor/layers/quantization/bitsandbytes.py b/vllm/model_executor/layers/quantization/bitsandbytes.py deleted file mode 100644 index ec3162c08a..0000000000 --- a/vllm/model_executor/layers/quantization/bitsandbytes.py +++ /dev/null @@ -1,614 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from functools import cached_property -from typing import Any, Union - -import torch -from packaging import version - -from vllm.model_executor.layers.fused_moe import ( - FusedMoEConfig, - FusedMoEMethodBase, - FusedMoEQuantConfig, - RoutedExperts, - SharedExperts, -) -from vllm.model_executor.layers.linear import ( - LinearBase, - LinearMethodBase, - UnquantizedLinearMethod, - set_weight_attrs, -) -from vllm.model_executor.layers.quantization import ( - QuantizationConfig, - QuantizationMethods, -) -from vllm.platforms import current_platform -from vllm.utils.torch_utils import direct_register_custom_op - - -def _check_bitsandbytes_version(): - min_version = "0.49.2" if current_platform.is_rocm() else "0.48.1" - try: - import bitsandbytes - - if version.parse(bitsandbytes.__version__) < version.parse(min_version): - raise ImportError( - "bitsandbytes version is wrong. Please " - f"install bitsandbytes>={min_version}." - ) - except ImportError as err: - raise ImportError( - f"Please install bitsandbytes>={min_version} via " - f"`pip install bitsandbytes>={min_version}` to use " - "bitsandbytes quantizer." - ) from err - - -class BitsAndBytesConfig(QuantizationConfig): - """Config class for BitsAndBytes Quantization. - - Reference: https://arxiv.org/abs/2305.14314 - """ - - def __init__( - self, - load_in_8bit: bool = False, - load_in_4bit: bool = True, - bnb_4bit_compute_dtype: str = "float32", - bnb_4bit_quant_storage: str = "uint8", - bnb_4bit_quant_type: str = "fp4", - bnb_4bit_use_double_quant: bool = False, - llm_int8_enable_fp32_cpu_offload: bool = False, - llm_int8_has_fp16_weight: bool = False, - llm_int8_skip_modules: list[str] | None = None, - llm_int8_threshold: float = 6.0, - ) -> None: - super().__init__() - self.load_in_8bit = load_in_8bit - self.load_in_4bit = load_in_4bit - self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype - self.bnb_4bit_quant_storage = bnb_4bit_quant_storage - self.bnb_4bit_quant_type = bnb_4bit_quant_type - self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant - self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload - self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight - self.llm_int8_skip_modules = llm_int8_skip_modules or [] - self.llm_int8_threshold = llm_int8_threshold - - if self.bnb_4bit_quant_storage not in ["uint8"]: - raise ValueError( - f"Unsupported bnb_4bit_quant_storage: {self.bnb_4bit_quant_storage}" - ) - - def __repr__(self) -> str: - return ( - f"BitsAndBytesConfig(load_in_8bit={self.load_in_8bit}, " - f"load_in_4bit={self.load_in_4bit}, " - f"bnb_4bit_compute_dtype={self.bnb_4bit_compute_dtype}, " - f"bnb_4bit_quant_storage={self.bnb_4bit_quant_storage}, " - f"bnb_4bit_quant_type={self.bnb_4bit_quant_type}, " - f"llm_int8_skip_modules={self.llm_int8_skip_modules})" - ) - - @classmethod - def get_name(self) -> QuantizationMethods: - return "bitsandbytes" - - @classmethod - def get_supported_act_dtypes(self) -> list[torch.dtype]: - return [torch.float32, torch.float16, torch.bfloat16] - - @classmethod - def get_min_capability(cls) -> int: - return 70 - - @staticmethod - def get_config_filenames() -> list[str]: - return [] - - @classmethod - def from_config(cls, config: dict[str, Any]) -> "BitsAndBytesConfig": - def get_safe_value(config, keys, default_value=None): - try: - value = cls.get_from_keys(config, keys) - return value if value is not None else default_value - except ValueError: - return default_value - - load_in_8bit = get_safe_value(config, ["load_in_8bit"], default_value=False) - load_in_4bit = get_safe_value(config, ["load_in_4bit"], default_value=True) - bnb_4bit_compute_dtype = get_safe_value( - config, ["bnb_4bit_compute_dtype"], default_value="float32" - ) - bnb_4bit_quant_storage = get_safe_value( - config, ["bnb_4bit_quant_storage"], default_value="uint8" - ) - bnb_4bit_quant_type = get_safe_value( - config, ["bnb_4bit_quant_type"], default_value="fp4" - ) - bnb_4bit_use_double_quant = get_safe_value( - config, ["bnb_4bit_use_double_quant"], default_value=False - ) - llm_int8_enable_fp32_cpu_offload = get_safe_value( - config, ["llm_int8_enable_fp32_cpu_offload"], default_value=False - ) - llm_int8_has_fp16_weight = get_safe_value( - config, ["llm_int8_has_fp16_weight"], default_value=False - ) - llm_int8_skip_modules = get_safe_value( - config, ["llm_int8_skip_modules"], default_value=[] - ) - llm_int8_threshold = get_safe_value( - config, ["llm_int8_threshold"], default_value=6.0 - ) - - return cls( - load_in_8bit=load_in_8bit, - load_in_4bit=load_in_4bit, - bnb_4bit_compute_dtype=bnb_4bit_compute_dtype, - bnb_4bit_quant_storage=bnb_4bit_quant_storage, - bnb_4bit_quant_type=bnb_4bit_quant_type, - bnb_4bit_use_double_quant=bnb_4bit_use_double_quant, - llm_int8_enable_fp32_cpu_offload=llm_int8_enable_fp32_cpu_offload, - llm_int8_has_fp16_weight=llm_int8_has_fp16_weight, - llm_int8_skip_modules=llm_int8_skip_modules, - llm_int8_threshold=llm_int8_threshold, - ) - - def get_quant_method( - self, layer: torch.nn.Module, prefix: str - ) -> Union["LinearMethodBase", "BitsAndBytesMoEMethod"] | None: - if isinstance(layer, LinearBase): - if is_layer_skipped_bnb(prefix, self.llm_int8_skip_modules): - return UnquantizedLinearMethod() - return BitsAndBytesLinearMethod(self) - elif isinstance(layer, RoutedExperts): - return BitsAndBytesMoEMethod(self, layer.moe_config) - return None - - -class BitsAndBytesWeightParameter(torch.nn.Parameter): - @cached_property - def dtype(self) -> torch.dtype: - return torch.get_default_dtype() - - -def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: list[str]): - # Split the prefix into its dot-separated components - components = prefix.split(".") - - # Check if any of the skip modules exactly matches any component - substr_check = any( - module_name in components for module_name in llm_int8_skip_modules - ) - - # Allow certain layers to not be quantized - set_components = set(".".join(components[: i + 1]) for i in range(len(components))) - set_llm_int8_skip_modules = set(llm_int8_skip_modules) - prefix_check = len(set_llm_int8_skip_modules & set_components) != 0 - - return substr_check or prefix_check - - -def calculate_quant_ratio(dtype): - if dtype.is_floating_point: - return torch.finfo(dtype).bits // torch.iinfo(torch.uint8).bits - else: - return torch.iinfo(dtype).bits // torch.iinfo(torch.uint8).bits - - -class BitsAndBytesLinearMethod(LinearMethodBase): - """Linear method for BitsAndBytes. - - Args: - quant_config: The BitsAndBytes quantization config. - """ - - def __init__(self, quant_config: BitsAndBytesConfig): - _check_bitsandbytes_version() - self.quant_config = quant_config - - def create_weights( - self, - layer: torch.nn.Module, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - from bitsandbytes.nn import Int8Params - - def create_qweight_for_8bit(): - qweight = Int8Params( - data=torch.empty( - sum(output_partition_sizes), - input_size_per_partition, - dtype=torch.int8, - ), - has_fp16_weights=self.quant_config.llm_int8_has_fp16_weight, - requires_grad=False, - ) - set_weight_attrs( - qweight, - { - "input_dim": 0, - "output_dim": 0, - "pack_factor": 1, - "use_bitsandbytes_8bit": True, - "generation": 0, - }, - ) - return qweight - - def create_qweight_for_4bit(): - quant_ratio = calculate_quant_ratio(params_dtype) - - total_size = input_size_per_partition * sum(output_partition_sizes) - if total_size % quant_ratio != 0: - raise ValueError( - "The input size is not aligned with the quantized weight shape." - ) - - qweight = BitsAndBytesWeightParameter( - torch.empty(total_size // quant_ratio, 1, dtype=torch.uint8), - requires_grad=False, - ) - set_weight_attrs( - qweight, - { - "input_dim": 0, - "output_dim": 0, - "pack_factor": quant_ratio, - "use_bitsandbytes_4bit": True, - }, - ) - return qweight - - if self.quant_config.load_in_8bit: - qweight = create_qweight_for_8bit() - else: - qweight = create_qweight_for_4bit() - # Enable parameters to have the same name as in the BNB - # checkpoint format. - layer.register_parameter("weight", qweight) - set_weight_attrs(qweight, extra_weight_attrs) - - def apply( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - if self.quant_config.load_in_8bit: - return self._apply_8bit_weight(layer, x, bias) - else: - return self._apply_4bit_weight(layer, x, bias) - - def _apply_8bit_weight( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - # only load the bitsandbytes module when needed - from bitsandbytes import MatmulLtState, matmul - - original_type = x.dtype - original_shape = x.shape - reshape_after_matmul = False - if x.ndim > 2: - x = x.reshape(-1, x.size(-1)) - reshape_after_matmul = True - bf_x = x.to(torch.bfloat16) - - qweight = layer.weight - offsets = qweight.bnb_shard_offsets - quant_states = qweight.bnb_quant_state - matmul_states = qweight.matmul_state - generation = qweight.generation - - out_dim_0 = x.shape[0] - out_dim_1 = sum( - [quant_state[1].shape[0] for quant_state in quant_states.items()] - ) - out = torch.empty(out_dim_0, out_dim_1, dtype=torch.float16, device=x.device) - - current_index = 0 - for i in range(len(quant_states)): - output_size = quant_states[i].shape[0] - - # in profile_run or the first generation of inference, - # create new matmul_states - if generation == 0 or generation == 1: - matmul_states[i] = MatmulLtState() - matmul_states[i].CB = qweight[offsets[i] : offsets[i + 1]] - matmul_states[i].SCB = quant_states[i].to(x.device) - matmul_states[i].threshold = self.quant_config.llm_int8_threshold - matmul_states[ - i - ].has_fp16_weights = self.quant_config.llm_int8_has_fp16_weight - matmul_states[i].is_training = False - if ( - matmul_states[i].threshold > 0.0 - and not matmul_states[i].has_fp16_weights - ): - matmul_states[i].use_pool = True - - new_x = bf_x.unsqueeze(0) - - out[:, current_index : current_index + output_size] = matmul( - new_x, qweight[offsets[i] : offsets[i + 1]], state=matmul_states[i] - ) - - current_index += output_size - - out = out.to(original_type) - - if reshape_after_matmul: - out = out.view(*original_shape[:-1], out.size(-1)) - - if bias is not None: - out += bias - - qweight.generation += 1 - - return out - - def _apply_4bit_weight( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - original_type = x.dtype - original_shape = x.shape - reshape_after_matmul = False - if x.ndim > 2: - x = x.reshape(-1, x.size(-1)) - reshape_after_matmul = True - bf_x = x.to(torch.bfloat16) - - qweight = layer.weight - quant_states = qweight.bnb_quant_state - offsets = qweight.bnb_shard_offsets - - out_dim_0 = x.shape[0] - out_dim_1 = sum( - [quant_state[1].shape[0] for quant_state in quant_states.items()] - ) - out = torch.empty(out_dim_0, out_dim_1, dtype=torch.bfloat16, device=x.device) - apply_bnb_4bit(bf_x, qweight, offsets, out) - out = out.to(original_type) - - if reshape_after_matmul: - out = out.view(*original_shape[:-1], out.size(-1)) - - if bias is not None: - out += bias - - return out - - -def _apply_bnb_4bit( - x: torch.Tensor, - weight: torch.Tensor, - offsets: torch.Tensor, - out: torch.Tensor, -) -> None: - # only load the bitsandbytes module when needed - from bitsandbytes import matmul_4bit - - quant_states = weight.bnb_quant_state - current_index = 0 - for i in range(len(quant_states)): - output_size = quant_states[i].shape[0] - # It is more efficient to use out kwarg like - # matmul_4bit(..., out = ...). Infeasible now due to the bug - # https://github.com/TimDettmers/bitsandbytes/issues/1235. - # Need to change after the bug is fixed. - out[:, current_index : current_index + output_size] = matmul_4bit( - x, weight[offsets[i] : offsets[i + 1]].t(), quant_states[i] - ) - current_index += output_size - - -def _apply_bnb_4bit_fake( - x: torch.Tensor, - weight: torch.Tensor, - offsets: torch.Tensor, - out: torch.Tensor, -) -> None: - return - - -try: - direct_register_custom_op( - op_name="apply_bnb_4bit", - op_func=_apply_bnb_4bit, - mutates_args=["out"], - fake_impl=_apply_bnb_4bit_fake, - dispatch_key=current_platform.dispatch_key, - ) - apply_bnb_4bit = torch.ops.vllm.apply_bnb_4bit - -except AttributeError as error: - raise error - - -class BitsAndBytesMoEMethod(FusedMoEMethodBase): - """MoE method for BitsAndBytes. - - Args: - quant_config: The BitsAndBytes quantization config. - """ - - def __init__( - self, - quant_config: BitsAndBytesConfig, - moe: FusedMoEConfig, - ): - super().__init__(moe) - _check_bitsandbytes_version() - self.quant_config = quant_config - - def create_weights( - self, - layer: RoutedExperts, - num_experts: int, - hidden_size: int, - intermediate_size_per_partition: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - if self.quant_config.load_in_8bit: - call_fun = self._create_weights_8bit - else: - call_fun = self._create_weights_4bit - call_fun( - layer, - num_experts, - hidden_size, - intermediate_size_per_partition, - params_dtype, - **extra_weight_attrs, - ) - - def get_fused_moe_quant_config( - self, layer: RoutedExperts - ) -> FusedMoEQuantConfig | None: - return None - - def apply( - self, - layer: RoutedExperts, - x: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - shared_experts: SharedExperts | None, - shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: - from vllm.model_executor.layers.fused_moe import fused_experts - - # TODO(bnell): Do these need to be called on the hot path? - if self.quant_config.load_in_8bit: - w13, w2 = self._apply_8bit_dequant(layer) - else: - w13, w2 = self._apply_4bit_dequnt(layer) - return fused_experts( - hidden_states=x, - w1=w13, - w2=w2, - topk_weights=topk_weights, - topk_ids=topk_ids, - activation=layer.activation, - apply_router_weight_on_input=layer.apply_router_weight_on_input, - global_num_experts=layer.global_num_experts, - expert_map=layer.expert_map, - quant_config=self.moe_quant_config, - ) - - def _create_weights_4bit( - self, - layer: torch.nn.Module, - num_experts: int, - hidden_size: int, - intermediate_size_per_partition: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - quant_ratio = calculate_quant_ratio(params_dtype) - # Fused gate_up_proj (column parallel) - w13_total_size = ( - hidden_size * self.moe.w13_num_shards * intermediate_size_per_partition - ) // quant_ratio - w13_qweight = torch.nn.Parameter( - torch.empty( - num_experts, - w13_total_size, - 1, - dtype=torch.uint8, - ), - requires_grad=False, - ) - layer.register_parameter("w13_weight", w13_qweight) - set_weight_attrs(w13_qweight, extra_weight_attrs) - set_weight_attrs( - w13_qweight, - { - "num_experts": num_experts, - "input_dim": hidden_size, - "output_dim": self.moe.w13_num_shards * intermediate_size_per_partition, - "experts_shape": ( - num_experts, - intermediate_size_per_partition * self.moe.w13_num_shards, - hidden_size, - ), - "pack_factor": quant_ratio, - "use_bitsandbytes_4bit": True, - }, - ) - # down_proj (row parallel) - w2_total_size = (hidden_size * intermediate_size_per_partition) // quant_ratio - w2_qweight = torch.nn.Parameter( - torch.empty( - num_experts, - w2_total_size, - 1, - dtype=torch.uint8, - ), - requires_grad=False, - ) - set_weight_attrs( - w2_qweight, - { - "num_experts": num_experts, - "input_dim": intermediate_size_per_partition, - "output_dim": hidden_size, - "experts_shape": ( - num_experts, - hidden_size, - intermediate_size_per_partition, - ), - "pack_factor": quant_ratio, - "use_bitsandbytes_4bit": True, - }, - ) - layer.register_parameter("w2_weight", w2_qweight) - set_weight_attrs(w2_qweight, extra_weight_attrs) - - def _create_weights_8bit( - self, - layer: torch.nn.Module, - num_experts: int, - hidden_size: int, - intermediate_size_per_partition: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - raise NotImplementedError - - def _apply_4bit_dequnt( - self, layer: torch.nn.Module - ) -> tuple[torch.Tensor, torch.Tensor]: - from bitsandbytes.functional import dequantize_4bit - - w13 = dequantize_4bit( - layer.w13_weight.reshape(-1, 1), - layer.w13_weight.bnb_quant_state, - ) - w2 = dequantize_4bit( - layer.w2_weight.reshape(-1, 1), - layer.w2_weight.bnb_quant_state, - ) - w13 = w13.reshape(layer.w13_weight.experts_shape) - w2 = w2.reshape(layer.w2_weight.experts_shape) - return w13, w2 - - def _apply_8bit_dequant( - self, layer: torch.nn.Module - ) -> tuple[torch.Tensor, torch.Tensor]: - raise NotImplementedError diff --git a/vllm/model_executor/model_loader/__init__.py b/vllm/model_executor/model_loader/__init__.py index 1ae78b77c0..a902c61da1 100644 --- a/vllm/model_executor/model_loader/__init__.py +++ b/vllm/model_executor/model_loader/__init__.py @@ -9,7 +9,6 @@ from vllm.config import ModelConfig, VllmConfig from vllm.config.load import LoadConfig from vllm.logger import init_logger from vllm.model_executor.model_loader.base_loader import BaseModelLoader -from vllm.model_executor.model_loader.bitsandbytes_loader import BitsAndBytesModelLoader from vllm.model_executor.model_loader.default_loader import DefaultModelLoader from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader from vllm.model_executor.model_loader.modelexpress_loader import ( @@ -33,7 +32,6 @@ logger = init_logger(__name__) LoadFormats = Literal[ "auto", "hf", - "bitsandbytes", "dummy", "fastsafetensors", "instanttensor", @@ -50,7 +48,6 @@ LoadFormats = Literal[ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = { "auto": DefaultModelLoader, "hf": DefaultModelLoader, - "bitsandbytes": BitsAndBytesModelLoader, "dummy": DummyModelLoader, "fastsafetensors": DefaultModelLoader, "instanttensor": DefaultModelLoader, @@ -150,7 +147,6 @@ __all__ = [ "get_model_cls", "register_model_loader", "BaseModelLoader", - "BitsAndBytesModelLoader", "ModelExpressModelLoader", "DefaultModelLoader", "DummyModelLoader", diff --git a/vllm/model_executor/model_loader/bitsandbytes_loader.py b/vllm/model_executor/model_loader/bitsandbytes_loader.py deleted file mode 100644 index 525de8d6a8..0000000000 --- a/vllm/model_executor/model_loader/bitsandbytes_loader.py +++ /dev/null @@ -1,834 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import fnmatch -import glob -import itertools -import math -import os -from collections.abc import Callable, Generator -from typing import Any - -import numpy as np -import torch -from packaging import version -from torch import nn -from transformers.utils import SAFE_WEIGHTS_INDEX_NAME - -from vllm.config import ModelConfig -from vllm.config.load import LoadConfig -from vllm.distributed import ( - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, -) -from vllm.logger import init_logger -from vllm.lora.utils import is_moe_model -from vllm.model_executor.layers.fused_moe import RoutedExperts -from vllm.model_executor.layers.linear import ( - LinearBase, - MergedColumnParallelLinear, - QKVParallelLinear, - ReplicatedLinear, - RowParallelLinear, -) -from vllm.model_executor.model_loader.base_loader import BaseModelLoader -from vllm.model_executor.model_loader.utils import ParamMapping -from vllm.model_executor.model_loader.weight_utils import ( - download_safetensors_index_file_from_hf, - download_weights_from_hf, - filter_duplicate_safetensors_files, - filter_files_not_needed_for_inference, - pt_weights_iterator, - safetensors_weights_iterator, -) -from vllm.model_executor.models import is_pooling_model -from vllm.model_executor.utils import ( - get_moe_expert_mapping, - get_packed_modules_mapping, - set_weight_attrs, -) -from vllm.platforms import current_platform -from vllm.transformers_utils.repo_utils import hf_api -from vllm.utils.torch_utils import set_default_torch_dtype - -logger = init_logger(__name__) - - -class BitsAndBytesModelLoader(BaseModelLoader): - """Model loader to load model weights with BitsAndBytes quantization.""" - - possible_config_file_names = ["adapter_config.json"] - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - - # Save the module names without sharding. - self.unsharded_weights_modules: list[str] = [] - # Save the module names that are sharded by column. - self.column_sharded_weights_modules: list[str] = [] - # Modules whose weights might have fused on disk - # we need their output_sizes to make shard in flight correctly with TP - self.maybe_fused_weights_modules: dict[str, list[int]] = {} - # Store all module names (from transformers) that support - # BNB quantization. - self.target_modules: list[str] = [] - self.tp_disabled_modules: list[str] = [] - # Store the mapping of expert parameters for MoE models. - self.expert_params_mapping: list[tuple[str, str, int, str]] = [] - # mapping weight names from transformers to vllm. - self.weight_mapper: Callable = lambda name: name - self.pre_quant: bool = False - self.load_8bit: bool = False - self.is_pool_model: bool = False - - def _get_weight_files( - self, - model_name_or_path: str, - allowed_patterns: list[str], - revision: str | None = None, - ) -> tuple[str, list[str], str]: - """Retrieve weight files. Download the files if necessary. - - Return the weight files and the file pattern.""" - is_local = os.path.isdir(model_name_or_path) - - if is_local: - for pattern in allowed_patterns: - weight_files = glob.glob(os.path.join(model_name_or_path, pattern)) - if weight_files: - return model_name_or_path, weight_files, pattern - else: - repo_files = hf_api().list_repo_files(repo_id=model_name_or_path) - for pattern in allowed_patterns: - matching_files = fnmatch.filter(repo_files, pattern) - if matching_files: - hf_folder = download_weights_from_hf( - model_name_or_path, - self.load_config.download_dir, - [pattern], - revision, - ignore_patterns=self.load_config.ignore_patterns, - ) - return ( - hf_folder, - glob.glob(os.path.join(hf_folder, pattern)), - pattern, - ) - - raise RuntimeError(f"No model weights found in: `{model_name_or_path}`") - - def _prepare_weights( - self, model_name_or_path: str, revision: str | None - ) -> tuple[list[str], bool]: - """Prepare weight files for the model.""" - - allowed_patterns = ["*.safetensors", "*.bin", "*.pt"] - - hf_folder, hf_weights_files, matched_pattern = self._get_weight_files( - model_name_or_path, allowed_patterns, revision - ) - - use_safetensors = matched_pattern == "*.safetensors" - is_local = os.path.isdir(model_name_or_path) - index_file = SAFE_WEIGHTS_INDEX_NAME - if use_safetensors: - # For models like Mistral-7B-Instruct-v0.3 - # there are both sharded safetensors files and a consolidated - # safetensors file. Using both breaks. - # Here, we download the `model.safetensors.index.json` and filter - # any files not found in the index. - if not is_local: - download_safetensors_index_file_from_hf( - model_name_or_path, - index_file, - cache_dir=self.load_config.download_dir, - revision=revision, - ) - hf_weights_files = filter_duplicate_safetensors_files( - hf_weights_files, hf_folder, index_file - ) - else: - hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files) - - if len(hf_weights_files) == 0: - raise RuntimeError( - f"Cannot find any model weights with `{model_name_or_path}`" - ) - - return hf_weights_files, use_safetensors - - def _hf_weight_iter(self, hf_weights_files, use_safetensors: bool): - def _maybe_pool_model(module_name: str): - # For pool model, we need to add the prefix `model.` - # for the weight name if possible. - if ( - self.is_pool_model - and self.target_modules[0].startswith("model.") - and not module_name.startswith("model.") - ): - return "model." + module_name - - return module_name - - if use_safetensors: - iterator = safetensors_weights_iterator( - hf_weights_files, - self.load_config.use_tqdm_on_load, - ) - else: - iterator = pt_weights_iterator( - hf_weights_files, - self.load_config.use_tqdm_on_load, - self.load_config.pt_load_map_location, - ) - for org_name, param in iterator: - # mapping weight names from transformers to vllm while preserving - # original names. - mapped_name = self.weight_mapper(org_name) - mapped_name = _maybe_pool_model(mapped_name) - - yield org_name, mapped_name, param - - def _get_quantized_weights_iterator( - self, - model_name_or_path: str, - revision: str | None, - ) -> tuple[Generator[tuple[str, torch.Tensor], None, None], dict[str, Any]]: - """Get an iterator to the model weights with bitsandbytes quantization, - as well as the quantization state dictionary.""" - - # only load the bitsandbytes module when needed - try: - import bitsandbytes - - if version.parse(bitsandbytes.__version__) < version.parse("0.46.1"): - raise ImportError( - "bitsandbytes version is wrong. Please " - "install bitsandbytes>=0.46.1." - ) - except ImportError as err: - raise ImportError( - "Please install bitsandbytes>=0.46.1 via " - "`pip install bitsandbytes>=0.46.1` to use " - "bitsandbytes quantizer." - ) from err - - hf_weights_files, use_safetensors = self._prepare_weights( - model_name_or_path, revision - ) - - quant_state_dict: dict[str, Any] = {} - - if self.pre_quant: - if self.load_8bit: - return self._quantized_8bit_generator( - hf_weights_files, use_safetensors, quant_state_dict - ), quant_state_dict - else: - return self._quantized_4bit_generator( - hf_weights_files, use_safetensors, quant_state_dict - ), quant_state_dict - - return self._unquantized_generator( - hf_weights_files, use_safetensors, quant_state_dict - ), quant_state_dict - - def _is_8bit_weight_name(self, weight_name: str): - quantized_suffix = {".scb", ".weight_format"} - return any(weight_name.lower().endswith(suffix) for suffix in quantized_suffix) - - def _is_4bit_weight_name(self, weight_name: str): - quantized_suffix = { - "absmax", - "quant_map", - "nested_absmax", - "nested_quant_map", - "bitsandbytes", - } - suffix = weight_name.split(".")[-1] - return any(q_suffix in suffix for q_suffix in quantized_suffix) - - def _quantized_8bit_generator( - self, hf_weights_files, use_safetensors, quant_state_dict - ) -> Generator: - for ( - org_weight_name, - mapped_weight_name, - weight_tensor, - ) in self._hf_weight_iter(hf_weights_files, use_safetensors): - if not mapped_weight_name.lower().endswith(".scb"): - continue - - weight_key = mapped_weight_name.lower().replace(".scb", ".weight") - quant_state_dict[weight_key] = weight_tensor - - for ( - org_weight_name, - mapped_weight_name, - weight_tensor, - ) in self._hf_weight_iter(hf_weights_files, use_safetensors): - if self._is_8bit_weight_name(mapped_weight_name): - continue - - if mapped_weight_name in quant_state_dict: - set_weight_attrs(weight_tensor, {"load_in_8bit": True}) - yield org_weight_name, weight_tensor - else: - yield org_weight_name, weight_tensor - - def _quantized_4bit_generator( - self, hf_weights_files, use_safetensors, quant_state_dict - ) -> Generator: - from bitsandbytes.functional import QuantState - - # First iterate over all quant state weights - weight_iterator = self._hf_weight_iter(hf_weights_files, use_safetensors) - temp_state_dict = {} - for ( - org_weight_name, - mapped_weight_name, - weight_tensor, - ) in weight_iterator: - if not self._is_4bit_weight_name(mapped_weight_name): - continue - # bitsandbytes library requires - # weight.quant_state.bitsandbytes__* in CPU - if "quant_state.bitsandbytes" in mapped_weight_name: - temp_state_dict[mapped_weight_name] = weight_tensor.cpu().data - else: - temp_state_dict[mapped_weight_name] = weight_tensor - - # Closure to parse quant_state for each prequant weight - def _parse_quant_state(param_name: str, temp_state_dict: dict) -> QuantState: - quant_state = {} - for k in temp_state_dict: - if param_name + "." in k: - quant_state[k] = temp_state_dict[k] - - return QuantState.from_dict( - quant_state, device=current_platform.device_type - ) - - # Second iterate over all prequant and normal weights - # pre quantized weights would have a quant_state - for ( - org_weight_name, - mapped_weight_name, - weight_tensor, - ) in self._hf_weight_iter(hf_weights_files, use_safetensors): - if self._is_4bit_weight_name(mapped_weight_name): - continue - - if ( - f"{mapped_weight_name}.quant_state.bitsandbytes__nf4" in temp_state_dict - ) or ( - f"{mapped_weight_name}.quant_state.bitsandbytes__fp4" in temp_state_dict - ): - quant_state = _parse_quant_state(mapped_weight_name, temp_state_dict) - quant_state_dict[mapped_weight_name] = quant_state - yield org_weight_name, weight_tensor - else: - yield org_weight_name, weight_tensor - - def _unquantized_generator( - self, hf_weights_files, use_safetensors, quant_state_dict - ) -> Generator: - from bitsandbytes.functional import quantize_4bit - - global_tp_size = get_tensor_model_parallel_world_size() - global_tp_rank = get_tensor_model_parallel_rank() - check_match = ( - lambda weight_name, module_name: weight_name.removesuffix(".weight") - == module_name - ) - for ( - org_weight_name, - mapped_weight_name, - weight_tensor, - ) in self._hf_weight_iter(hf_weights_files, use_safetensors): - # override tp_size and tp_rank if the module has disabled TP - if any( - tp_disabled_module in mapped_weight_name - for tp_disabled_module in self.tp_disabled_modules - ): - tp_size = 1 - tp_rank = 0 - else: - tp_size = global_tp_size - tp_rank = global_tp_rank - - if any( - target_module in mapped_weight_name - for target_module in self.target_modules - ) and mapped_weight_name.endswith(".weight"): - # Without sharding - if any( - check_match(mapped_weight_name, module) - for module in self.unsharded_weights_modules - ): - weight_sub_tensor = weight_tensor - # Shard by column - elif any( - check_match(mapped_weight_name, module) - for module in self.column_sharded_weights_modules - ): - total_size = weight_tensor.size(-1) - start_index = total_size // tp_size * tp_rank - end_index = total_size // tp_size * (tp_rank + 1) - weight_sub_tensor = weight_tensor[..., start_index:end_index] - # Weights have fused on disk. In this case, we assume that the - # weight and module use same name. - elif any( - check_match(mapped_weight_name, module) - for module in self.maybe_fused_weights_modules - ): - # special case for fused weights - # get the size of each shard weight tensor - total_shard_sizes = next( - ( - sizes - for module, sizes in self.maybe_fused_weights_modules.items() # noqa: E501 - if check_match(mapped_weight_name, module) - ) - ) - total_size = weight_tensor.size(0) - assert total_size == sum(total_shard_sizes) - # get the start/end index of each shard weight tensor - total_start_index = list( - itertools.accumulate([0] + total_shard_sizes) - )[:-1] - shard_weights_index = [ - ( - idx + size // tp_size * tp_rank, - idx + size // tp_size * (tp_rank + 1), - ) - for idx, size in zip(total_start_index, total_shard_sizes) - ] - # slice and reorder the weight tensor - weight_tensor = [ - weight_tensor[start_index:end_index, ...] - for start_index, end_index in shard_weights_index - ] - weight_sub_tensor = torch.cat(weight_tensor, dim=0) - # Shard by row - else: - total_size = weight_tensor.size(0) - start_index = total_size // tp_size * tp_rank - end_index = total_size // tp_size * (tp_rank + 1) - weight_sub_tensor = weight_tensor[start_index:end_index, ...] - - # bitsandbytes requires data in GPU - if weight_sub_tensor.is_cuda: - loaded_weight = weight_sub_tensor - else: - loaded_weight = weight_sub_tensor.to( - device=current_platform.device_type - ) - - # remove the following after the issue is fixed: - # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1342 - if loaded_weight.is_contiguous() is False: - loaded_weight = loaded_weight.contiguous() - - with set_default_torch_dtype(torch.float32): - processed_weight, quant_state = quantize_4bit( - loaded_weight, - compress_statistics=True, - quant_type="nf4", - ) - - quant_state_dict[mapped_weight_name] = quant_state - else: - processed_weight = weight_tensor - yield org_weight_name, processed_weight - - def _get_bnb_target_modules(self, model: nn.Module) -> None: - """ - Identify and collect all modules that support BitsAndBytes - quantization. - """ - for name, module in model.named_modules(): - if isinstance(module, LinearBase) and hasattr( - module.quant_method, "quant_config" - ): - if modules_info := self.modules_mapping.get_sub_modules(name): - # Map vllm's names to transformers's names. - rep_name, sub_modules = modules_info - for sub_name in sub_modules: - new_name = name.replace(rep_name, sub_name) - self.target_modules.append(new_name) - if module.disable_tp: - self.tp_disabled_modules.append(new_name) - # Add original module name even if the module has stacked map, - # in case model has a mixture of disk-merged and disk-split - # weights with same last name. - self.target_modules.append(name) - if module.disable_tp: - self.tp_disabled_modules.append(name) - elif isinstance(module, RoutedExperts) and hasattr( - module.quant_method, "quant_config" - ): - # TODO: support RoutedExperts with prequant and 8bit. - if self.pre_quant and self.load_8bit: - raise ValueError( - "Prequant BitsAndBytes 8bit models with RoutedExperts " - "is not supported yet." - ) - # Get the corresponding weight name using module name and - # expert_params_mapping. - - for exp in self.expert_params_mapping: - weight_name = exp[1] - rep_name = name.replace("experts", "") + weight_name.removesuffix( - "." - ) - self.target_modules.append(rep_name) - - assert self.target_modules, ( - "vLLM currently does not support BNB quantization for" - ) - f" {type(model).__name__}" - - def _classify_module_sharding(self, model: nn.Module): - """ - Categorize modules based on their weight sharding requirements - for tensor parallelism. - """ - for name, module in model.named_modules(): - # Some modules like `ReplicatedLinear` should not have their weights - # sharded. The reason for implementing it this way is to avoid new - # static variable in the model implementation. - if isinstance(module, (ReplicatedLinear,)): - self.unsharded_weights_modules.append(name) - # `QKVParallelLinear` and `MergedColumnParallelLinear` might have - # fused weights on disk. We need to use the output sizes of these - # modules to shard the weights correctly. - elif isinstance(module, (QKVParallelLinear, MergedColumnParallelLinear)): - self.maybe_fused_weights_modules[name] = module.output_sizes - # In TP, these weights are partitioned along the column - # dimension (dim=-1) - elif isinstance(module, (RowParallelLinear,)): - self.column_sharded_weights_modules.append(name) - elif isinstance(module, RoutedExperts): - expert_mapping = self.expert_params_mapping - for exp in expert_mapping: - if exp[-1] == "w2": - weight_name = exp[1] - rep_name = name.replace( - "experts", "" - ) + weight_name.removesuffix(".") - self.column_sharded_weights_modules.append(rep_name) - - def _verify_model_compatibility( - self, model: nn.Module, model_config: ModelConfig - ) -> None: - """ - Verify that the model is compatible with BitsAndBytes quantization. - """ - if not hasattr(model, "load_weights"): - raise AttributeError( - "The required method 'load_weights' is not defined in class" - f" {type(model).__name__}." - ) - - if not hasattr(model, "packed_modules_mapping"): - raise AttributeError( - f"Model {type(model).__name__} does not support BitsAndBytes " - "quantization yet. No 'packed_modules_mapping' found." - ) - - quant_config = getattr(model_config.hf_config, "quantization_config", None) - if quant_config and (quant_method := quant_config.get("quant_method")): - if quant_method == "bitsandbytes": - self.pre_quant = True - else: - raise ValueError( - f"BitsAndBytes loader does not support {quant_method} quantization" - ) - - # The quant_states in pre_quantized models cannot work with a split - # weight tensor. So TP does not work with pre_quantized bnb models. - if self.pre_quant and get_tensor_model_parallel_world_size() > 1: - raise ValueError( - "Prequant BitsAndBytes models with tensor parallelism is not " - "supported. Please try with pipeline parallelism." - ) - if quant_config and self.pre_quant: - self.load_8bit = quant_config.get("load_in_8bit", False) - - def _initialize_loader_state( - self, model: nn.Module, model_config: ModelConfig - ) -> None: - """ - Initialize the loader's internal state based on the model and - configuration. - """ - self.is_pool_model = is_pooling_model(model) - self.modules_mapping = ParamMapping(get_packed_modules_mapping(model)) - - if is_moe_model(model): - self.expert_params_mapping = get_moe_expert_mapping(model) - # For some models like Molmo, we need to use hf_to_vllm_mapper - # to ensure correct loading of weights. - if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None): - unstacked_mapper = hf_to_vllm_mapper.get_unstacked_mapper() - self.weight_mapper = lambda name, m=unstacked_mapper: m._map_name(name) - - self._get_bnb_target_modules(model) - self._classify_module_sharding(model) - - def _dequantize_dq(self, quant_states: Any): - """ - When BNB employs Double Quantization, we perform the dequantization of - these constants during weight loading rather than at inference time, - thereby avoiding this computational overhead during inference. This - comes at the cost of increased memory usage. - """ - from bitsandbytes.functional import QuantState, dequantize_blockwise - - def _dequantize_single_state(quant_state): - """Helper function to dequantize a single QuantState object.""" - if not (isinstance(quant_state, QuantState) and quant_state.nested): - return - - # Copied from: https://github.com/bitsandbytes-foundation/bitsandbytes/blob/0.45.3/bitsandbytes/functional.py#L1352-#L1356 - absmax = dequantize_blockwise(quant_state.absmax, quant_state.state2) - absmax += quant_state.offset - - # Ensure float32 dtype - if absmax.dtype != torch.float32: - absmax = absmax.float() - - quant_state.absmax = absmax - quant_state.nested = False - quant_state.offset = None - quant_state.state2 = None - - if isinstance(quant_states, dict): - for quant_state in quant_states.values(): - _dequantize_single_state(quant_state) - else: - _dequantize_single_state(quant_states) - return quant_states - - def _fuse_moe_quant_states(self, model: nn.Module, quant_states_dict: dict) -> dict: - """ - - This function consolidates individual expert quantization states into - fused representations for w13 and w2. - """ - from bitsandbytes.functional import QuantState - - if not self.expert_params_mapping: - return dict() - - expert_mapping = self.expert_params_mapping - expert_qs_dict = {} - for name, module in model.named_modules(): - if not isinstance(module, RoutedExperts): - continue - w1_states_lst = [] - w2_states_lst = [] - w3_states_lst = [] - for exp in expert_mapping: - shard_id = exp[-1] - if shard_id not in ("w1", "w2", "w3"): - raise ValueError( - f"shard_id must be ['w1','w2','w3'] but got {shard_id}." - ) - layer_prefix = name.split("experts")[0] - weight_qual_name = layer_prefix + exp[1] + "weight" - quant_state = self._dequantize_dq(quant_states_dict[weight_qual_name]) - if shard_id == "w1": - w1_states_lst.append(quant_state) - elif shard_id == "w2": - w2_states_lst.append(quant_state) - else: - w3_states_lst.append(quant_state) - del quant_states_dict[weight_qual_name] - assert len(w1_states_lst) == len(w2_states_lst) == len(w3_states_lst) - w13_absmax_lst = [] - w2_absmax_lst = [] - w13_total_dim0 = 0 - w2_total_dim0 = 0 - for w1_qs, w2_qs, w3_qs in zip(w1_states_lst, w2_states_lst, w3_states_lst): - assert w1_qs.shape == w3_qs.shape - assert w1_qs.blocksize == w2_qs.blocksize == w3_qs.blocksize - assert w1_qs.dtype == w2_qs.dtype == w3_qs.dtype - # w1 and w3 are interleaved in storage - w13_absmax_lst.append(w1_qs.absmax) - w13_absmax_lst.append(w3_qs.absmax) - w2_absmax_lst.append(w2_qs.absmax) - w13_total_dim0 += w1_qs.shape[0] + w3_qs.shape[0] - w2_total_dim0 += w2_qs.shape[0] - - w13_absmax = torch.cat(w13_absmax_lst) - w2_absmax = torch.cat(w2_absmax_lst) - # Create fused quantization state for w13. - w13_qs = QuantState( - absmax=w13_absmax, - shape=(w13_total_dim0, w1_states_lst[0].shape[1]), - code=w1_states_lst[0].code, - blocksize=w1_states_lst[0].blocksize, - quant_type="nf4", - dtype=w1_states_lst[0].dtype, - ) - # Create fused quantization state for w2. - w2_qs = QuantState( - absmax=w2_absmax, - shape=(w2_total_dim0, w2_states_lst[0].shape[1]), - code=w2_states_lst[0].code, - blocksize=w2_states_lst[0].blocksize, - quant_type="nf4", - dtype=w2_states_lst[0].dtype, - ) - # The weight suffixes .w13_weight and .w2_weight are consistent - # with the param in BitsAndBytesMoEMethod. - w13_weight_name = name + ".w13_weight" - w2_weight_name = name + ".w2_weight" - expert_qs_dict[w13_weight_name] = w13_qs - expert_qs_dict[w2_weight_name] = w2_qs - return expert_qs_dict - - def _stack_quantization_states( - self, model: nn.Module, quant_state_dict: dict - ) -> dict[str, dict[int, Any]]: - stacked_quant_state_dict: dict[str, dict[int, Any]] = {} - # TODO: Change this lazy import to normal import - # after the checks are updated to run on a new version - from vllm.model_executor.models.utils import is_pp_missing_parameter - - param_dict = dict(model.named_parameters()) - for quant_param_name in quant_state_dict: - if is_pp_missing_parameter(quant_param_name, model): - continue - - non_stacked_param_name = quant_param_name - - shard_index = 0 - for shard_name, ( - weight_name, - index, - ) in self.modules_mapping.inverse_packed_mapping.items(): - # Some models, such as MiniCPM V2.5/2.6, contain both - # module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj' - # from being incorrectly identified as being present in - # 'vpm.encoder.layers.0.self_attn.qkv_proj.weight - shard_pos = quant_param_name.find(shard_name) - can_correct_rename = (shard_pos > 0) and ( - quant_param_name[shard_pos - 1] == "." - ) - # If the quant_param_name is packed, it won't occur in the - # param_dict before renaming. - new_quant_param_name = quant_param_name.replace(shard_name, weight_name) - need_rename = (quant_param_name not in param_dict) and ( - new_quant_param_name in param_dict - ) - if can_correct_rename and need_rename: - shard_index = index - quant_param_name = new_quant_param_name - break - - # Models like Clip/Siglip may skip some layers in initialization, - # causing unused quant_param_name in state_dict. - if quant_param_name not in param_dict: - continue - - if quant_param_name not in stacked_quant_state_dict: - stacked_quant_state_dict[quant_param_name] = {} - - stacked_quant_state_dict[quant_param_name][shard_index] = quant_state_dict[ - non_stacked_param_name - ] - - # repeat k_proj for v_proj for k_eq_v models (e.g. Gemma4) - config = getattr(model, "config", None) - if config is not None: - text_config = config.get_text_config() - if getattr(text_config, "attention_k_eq_v", False): - shard_packed = { - name - for name, subs in self.modules_mapping.packed_mapping.items() - if len(subs) == 3 - } - for param_name, shards in stacked_quant_state_dict.items(): - is_target = ( - isinstance(shards, dict) - and len(shards) == 2 - and any( - param_name.endswith(f"{p}.weight") for p in shard_packed - ) - ) - if is_target: - assert 1 in shards and 2 not in shards - shards[2] = shards[1] - - return stacked_quant_state_dict - - def _bind_quant_states_to_params( - self, model: nn.Module, stacked_quant_state_dict: dict - ) -> None: - # save quant_states and offsets as the attributes of the parameters - param_dict = dict(model.named_parameters()) - for param_name, param in param_dict.items(): - if param_name in stacked_quant_state_dict: - quant_states = stacked_quant_state_dict[param_name] - # Dequantize double quantized values during weight loading. - self._dequantize_dq(quant_states) - set_weight_attrs(param, {"bnb_quant_state": quant_states}) - if not isinstance(quant_states, dict): - continue - - pack_ratio = getattr(param, "pack_factor", -1) - if pack_ratio == -1: - raise ValueError(f"pack_factor not set for parameter {param_name}.") - - num_elements = [0] * len(quant_states) - for seq, quant_state in quant_states.items(): - num_elements[seq] = math.prod(quant_state.shape) // pack_ratio - - offsets = np.concatenate(([0], np.cumsum(num_elements))) - # Make torch infer_schema happy - offsets = torch.tensor(offsets).cpu() - set_weight_attrs(param, {"bnb_shard_offsets": offsets}) - - if self.load_8bit: - set_weight_attrs( - param, {"matmul_state": [None] * len(quant_states)} - ) - - def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None: - self._verify_model_compatibility(model, model_config) - self._initialize_loader_state(model, model_config) - - logger.info( - "Loading weights with BitsAndBytes quantization. May take a while ..." - ) - qweight_iterator, quant_state_dict = self._get_quantized_weights_iterator( - model_config.model, - model_config.revision, - ) - weights_to_load = {name for name, _ in model.named_parameters()} - loaded_weights = model.load_weights(qweight_iterator) - # Some models may have weights loading tracker unimplemented. - if loaded_weights is not None: - weights_not_loaded = weights_to_load - loaded_weights - if weights_not_loaded: - raise ValueError( - "Following weights were not initialized from " - f"checkpoint: {weights_not_loaded}" - ) - expert_quant_state_dict = self._fuse_moe_quant_states(model, quant_state_dict) - - stacked_quant_state_dict = self._stack_quantization_states( - model, quant_state_dict - ) - - stacked_quant_state_dict = { - **expert_quant_state_dict, - **stacked_quant_state_dict, - } - self._bind_quant_states_to_params(model, stacked_quant_state_dict) - torch.accelerator.empty_cache() - - def download_model(self, model_config: ModelConfig) -> None: - self._prepare_weights(model_config.model, model_config.revision) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 36683225d1..a472c75435 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -5,7 +5,6 @@ import inspect import warnings from contextlib import contextmanager -from dataclasses import dataclass, field from typing import Any import torch @@ -263,35 +262,6 @@ def get_architecture_class_name(model_config: ModelConfig) -> str: return get_model_architecture(model_config)[1] -@dataclass -class ParamMapping: - """ - A class to handle parameter mapping for model weight loading. - It creates a bidirectional mapping between packed parameters and their - constituent parts. - """ - - packed_mapping: dict[str, list[str]] - inverse_packed_mapping: dict[str, tuple[str, int]] = field(default_factory=dict) - - def __post_init__(self): - for packed_name, sub_params in self.packed_mapping.items(): - # Skip self-contained cases (e.g., {"W_pack": ["W_pack"]}) - if len(sub_params) == 1 and sub_params[0] == packed_name: - continue - for index, param_name in enumerate(sub_params): - self.inverse_packed_mapping[param_name] = ( - packed_name, - index, - ) - - def get_sub_modules(self, module_name: str) -> tuple[str, list[str]] | None: - for key, value in self.packed_mapping.items(): - if module_name.endswith(key): - return key, value - return None - - def configure_quant_config( quant_config: QuantizationConfig, model_class: type[nn.Module] ): diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index e1897a77f1..167f923759 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -330,9 +330,6 @@ def get_quant_config( assert isinstance(model_config.quantization_config, QuantizationConfigArgs) return OnlineQuantizationConfig(args=model_config.quantization_config) - # Inflight BNB quantization - if model_config.quantization == "bitsandbytes": - return quant_cls.from_config({}) model_name_or_path = ( maybe_download_from_modelscope( model_config.model, @@ -380,9 +377,7 @@ def get_quant_config( with open(quant_config_file) as f: config = json.load(f) - if model_config.quantization == "bitsandbytes": - config["adapter_name_or_path"] = model_config.model - elif model_config.quantization in ("modelopt", "modelopt_mixed"): + if model_config.quantization in ("modelopt", "modelopt_mixed"): if config.get("producer", {}).get("name") == "modelopt": return quant_cls.from_config(config) else: diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index 6528b8488c..348c0578f3 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -699,10 +699,6 @@ class OpenPanguSinkAttention(nn.Module): output_dim = getattr(param, "output_dim", None) is_sharded_weight = getattr(param, "is_sharded_weight", False) - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) - # bitsandbytes loads the weights of the specific portion - # no need to narrow - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit param_data = param.data if output_dim is not None and not is_sharded_weight: diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index dd577e1cd2..83c95edbf6 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -524,7 +524,6 @@ class RocmPlatform(Platform): "mxfp4", "mxfp8", "torchao", - "bitsandbytes", "modelopt", "modelopt_fp4", "modelopt_mxfp8", diff --git a/vllm/v1/metrics/perf.py b/vllm/v1/metrics/perf.py index 66a600af14..75524ef128 100644 --- a/vllm/v1/metrics/perf.py +++ b/vllm/v1/metrics/perf.py @@ -46,7 +46,7 @@ class InvalidComponent(Exception): # FfnQuantizationConfigParser to determine the weight_byte_size for # flops/memory estimation. # -# NOTE: Methods like GPTQ and BitsAndBytes support variable bit-widths +# NOTE: Methods like GPTQ support variable bit-widths # (e.g., 4-bit and 8-bit). We default to 4-bit (0.5 bytes) since this # is by far the most common configuration. _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = { @@ -63,7 +63,6 @@ _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = { "awq_marlin": 0.5, "gptq": 0.5, "gptq_marlin": 0.5, - "bitsandbytes": 0.5, "modelopt_fp4": 0.5, "petit_nvfp4": 0.5, "compressed-tensors": 0.5,