From 7c50959fe6ff7ea5a92c582b3fb26e79852f90f5 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Fri, 5 Jan 2024 17:11:26 +0800 Subject: [PATCH] fix #2098 Former-commit-id: f6fdd83f8a6bf3e48cf08fd098ec7b08d34d16d7 --- src/llmtuner/model/patcher.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/llmtuner/model/patcher.py b/src/llmtuner/model/patcher.py index 426372ee..d3ec0bb1 100644 --- a/src/llmtuner/model/patcher.py +++ b/src/llmtuner/model/patcher.py @@ -139,7 +139,9 @@ def _configure_quantization( raise ValueError("DeepSpeed ZeRO-3 is incompatible with quantization.") config_kwargs["device_map"] = {"": get_current_device()} - quantization_config = getattr(config, "quantization_config", None) + quantization_config: Dict[str, Any] = getattr(config, "quantization_config", None) + if quantization_config.get("quant_method", None) == "gptq" and quantization_config.get("bits", -1) == 4: + quantization_config["use_exllama"] = False # disable exllama logger.info("Loading {}-bit pre-quantized model.".format(quantization_config.get("bits", -1))) elif model_args.export_quantization_bit is not None: # gptq