[Doc] Move examples into categories (#11840)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -452,9 +452,9 @@ class TensorizerLoader(BaseModelLoader):
|
||||
"""Load a serialized model with tensorizer to the CPU.
|
||||
|
||||
This is only necessary when the model isn't vLLM-tensorized (see
|
||||
examples/tensorize_vllm_model.py) This should still be faster than
|
||||
default HuggingFace loading, but will be slower than loading a
|
||||
vLLM-tensorized model.
|
||||
examples/other/tensorize_vllm_model.py) This should still
|
||||
be faster than default HuggingFace loading, but will be slower than
|
||||
loading a vLLM-tensorized model.
|
||||
"""
|
||||
device_config = vllm_config.device_config
|
||||
model_config = vllm_config.model_config
|
||||
@@ -472,7 +472,7 @@ class TensorizerLoader(BaseModelLoader):
|
||||
"""Load a serialized model with tensorizer.
|
||||
|
||||
Expects a vLLM-tensorized model. See the
|
||||
examples/tensorize_vllm_model.py example script
|
||||
examples/other/tensorize_vllm_model.py example script
|
||||
for serializing vLLM models."""
|
||||
|
||||
device_config = vllm_config.device_config
|
||||
@@ -529,7 +529,8 @@ class ShardedStateLoader(BaseModelLoader):
|
||||
Model loader that directly loads each worker's model state dict, which
|
||||
enables a fast load path for large tensor-parallel models where each worker
|
||||
only needs to read its own shard rather than the entire checkpoint. See
|
||||
`examples/save_sharded_state.py` for creating a sharded checkpoint.
|
||||
`examples/offline_inference/save_sharded_state.py` for creating a sharded
|
||||
checkpoint.
|
||||
"""
|
||||
|
||||
DEFAULT_PATTERN = "model-rank-{rank}-part-{part}.safetensors"
|
||||
|
||||
Reference in New Issue
Block a user