423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 19:06:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

add cal_lr.py

Browse Source 
Former-commit-id: 42c8fc4fb9
This commit is contained in:
hiyouga
2023-11-14 20:58:37 +08:00
parent 8387f3011c
commit fffb8ea764
4 changed files with 67 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/llmtuner/tuner/core/loader.py
View File

@@ -84,10 +84,9 @@ def load_model_and_tokenizer(
tokenizer._pad = MethodType(PreTrainedTokenizerBase._pad, tokenizer)
# Set model dtype
if model_args.compute_dtype is not None: # for training
setattr(config, "torch_dtype", model_args.compute_dtype)
else: # for evaluation, priority: bf16 > fp16 > fp32
if model_args.compute_dtype is None: # priority: bf16 > fp16 > fp32
model_args.compute_dtype = infer_optim_dtype(model_dtype=getattr(config, "torch_dtype", None))
setattr(config, "torch_dtype", model_args.compute_dtype)
# Fix config (for Qwen)
if getattr(config, "model_type", None) == "qwen":