423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-27 08:12:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix #2098

Browse Source 
Former-commit-id: f6fdd83f8a6bf3e48cf08fd098ec7b08d34d16d7
This commit is contained in:
hiyouga 2024-01-05 17:11:26 +08:00
parent a78cd5c5ad
commit 7c50959fe6
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/model/patcher.py
View File

@ -139,7 +139,9 @@ def _configure_quantization(
raise ValueError("DeepSpeed ZeRO-3 is incompatible with quantization.")
config_kwargs["device_map"] = {"": get_current_device()}
quantization_config = getattr(config, "quantization_config", None)
quantization_config: Dict[str, Any] = getattr(config, "quantization_config", None)
if quantization_config.get("quant_method", None) == "gptq" and quantization_config.get("bits", -1) == 4:
quantization_config["use_exllama"] = False # disable exllama
logger.info("Loading {}-bit pre-quantized model.".format(quantization_config.get("bits", -1)))
elif model_args.export_quantization_bit is not None: # gptq