From bd611e0090322fd999c40187219ed55fb0b86b2d Mon Sep 17 00:00:00 2001 From: hiyouga Date: Sat, 12 Aug 2023 22:00:01 +0800 Subject: [PATCH] fix rope scaling Former-commit-id: 8545c11c45906b33c78e144c2338963eaf0406b8 --- src/llmtuner/tuner/core/loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llmtuner/tuner/core/loader.py b/src/llmtuner/tuner/core/loader.py index 32d6247a..47c72de1 100644 --- a/src/llmtuner/tuner/core/loader.py +++ b/src/llmtuner/tuner/core/loader.py @@ -102,11 +102,11 @@ def load_model_and_tokenizer( ) current_max_length = getattr(config, "max_position_embeddings", None) - if current_max_length and model_args.model_max_length <= current_max_length: + if current_max_length and model_args.model_max_length > current_max_length: + scaling_factor = float(math.ceil(model_args.model_max_length / current_max_length)) + else: logger.warning("Input length is smaller than max length. Consider increase input length.") scaling_factor = 1.0 - else: - scaling_factor = float(math.ceil(model_args.model_max_length / current_max_length)) else: scaling_factor = 2.0