From 038d524fcd6cb946f32e7ba5792f86cd00387c6d Mon Sep 17 00:00:00 2001 From: hiyouga Date: Tue, 16 Apr 2024 18:09:31 +0800 Subject: [PATCH] Update parser.py Former-commit-id: c00f0771a5ab2a422e0300dcb6d88e8609a3b997 --- src/llmtuner/hparams/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llmtuner/hparams/parser.py b/src/llmtuner/hparams/parser.py index a9f8ffd7..03ab0c50 100644 --- a/src/llmtuner/hparams/parser.py +++ b/src/llmtuner/hparams/parser.py @@ -174,9 +174,6 @@ def get_train_args(args: Optional[Dict[str, Any]] = None) -> _TRAIN_CLS: ): raise ValueError("Distributed training does not support layer-wise GaLore.") - if finetuning_args.use_galore and training_args.deepspeed is not None: - raise ValueError("GaLore is incompatible with DeepSpeed yet.") - if ( finetuning_args.use_badam and finetuning_args.badam_mode == "layer" @@ -184,6 +181,9 @@ def get_train_args(args: Optional[Dict[str, Any]] = None) -> _TRAIN_CLS: ): raise ValueError("Layer-wise BAdam does not yet support distributed training, use ratio-wise BAdam.") + if (finetuning_args.use_galore or finetuning_args.use_badam) and training_args.deepspeed is not None: + raise ValueError("GaLore and BAdam are incompatible with DeepSpeed yet.") + if model_args.infer_backend == "vllm": raise ValueError("vLLM backend is only available for API, CLI and Web.")