423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-20 22:00:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix unusual output of 8bit models #278 #391

Browse Source 
Former-commit-id: 337ce5272b81f5561162beb08814b0e5abf23703
This commit is contained in:
hiyouga
2023-08-12 00:25:29 +08:00
parent 98aa629843
commit 9644286027
2 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/tuner/core/loader.py
View File

@@ -92,7 +92,7 @@ def load_model_and_tokenizer(
)
is_mergeable = False
config_kwargs["device_map"] = {"": int(os.environ.get("LOCAL_RANK", "0"))}
config_kwargs["device_map"] = {"": int(os.environ.get("LOCAL_RANK", "0"))} if is_trainable else "auto"
logger.info("Quantizing model to {} bit.".format(model_args.quantization_bit))
# Load and prepare pretrained models (without valuehead).