423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-15 16:18:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

tiny fix

Browse Source 
Former-commit-id: 50a34c043de6d9e1410291e1d8c1ea9d53754e9e
This commit is contained in:
hiyouga 2023-08-12 22:02:43 +08:00
parent c95479dddb
commit a22982f2fa
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/llmtuner/tuner/core/loader.py
View File

@ -83,17 +83,17 @@ def load_model_and_tokenizer(
# Set RoPE scaling # Set RoPE scaling
if model_args.rope_scaling is not None: if model_args.rope_scaling is not None:
require_version("transformers>=4.31.0", "RoPE scaling requires transformers>=4.31.0")
if hasattr(config, "use_dynamic_ntk"): # for Qwen models if hasattr(config, "use_dynamic_ntk"): # for Qwen models
if is_trainable: if is_trainable:
logger.warning("Qwen model does not support rope scaling in training.") logger.warning("Qwen model does not support RoPE scaling in training.")
else: else:
setattr(config, "use_dynamic_ntk", True) setattr(config, "use_dynamic_ntk", True)
setattr(config, "use_logn_attn", True) setattr(config, "use_logn_attn", True)
logger.info("Using dynamic NTK scaling.") logger.info("Using dynamic NTK scaling.")
elif hasattr(config, "rope_scaling"): # for LLaMA models elif hasattr(config, "rope_scaling"): # for LLaMA models
require_version("transformers>=4.31.0", "RoPE scaling requires transformers>=4.31.0")
if is_trainable: if is_trainable:
if model_args.rope_scaling == "dynamic": if model_args.rope_scaling == "dynamic":
logger.warning( logger.warning(