release v0.5.3

2025-12-16 20:00:36 +08:00 · 2024-02-29 00:34:19 +08:00
parent 804c1e7083
commit fa5ab21ebc
10 changed files with 116 additions and 67 deletions
--- a/src/llmtuner/webui/components/train.py
+++ b/src/llmtuner/webui/components/train.py
@@ -34,7 +34,7 @@ def create_train_tab(engine: "Engine") -> Dict[str, "Component"]:
    elem_dict.update(dict(training_stage=training_stage, dataset_dir=dataset_dir, dataset=dataset, **preview_elems))

    with gr.Row():
-        cutoff_len = gr.Slider(value=1024, minimum=4, maximum=8192, step=1)
+        cutoff_len = gr.Slider(value=1024, minimum=4, maximum=16384, step=1)
        learning_rate = gr.Textbox(value="5e-5")
        num_train_epochs = gr.Textbox(value="3.0")
        max_samples = gr.Textbox(value="100000")
@@ -52,8 +52,8 @@ def create_train_tab(engine: "Engine") -> Dict[str, "Component"]:
    )

    with gr.Row():
-        batch_size = gr.Slider(value=4, minimum=1, maximum=1024, step=1)
-        gradient_accumulation_steps = gr.Slider(value=4, minimum=1, maximum=1024, step=1)
+        batch_size = gr.Slider(value=2, minimum=1, maximum=1024, step=1)
+        gradient_accumulation_steps = gr.Slider(value=8, minimum=1, maximum=1024, step=1)
        lr_scheduler_type = gr.Dropdown(choices=[scheduler.value for scheduler in SchedulerType], value="cosine")
        max_grad_norm = gr.Textbox(value="1.0")
        val_size = gr.Slider(value=0, minimum=0, maximum=1, step=0.001)
@@ -122,25 +122,31 @@ def create_train_tab(engine: "Engine") -> Dict[str, "Component"]:

    with gr.Accordion(label="LoRA config", open=False) as lora_tab:
        with gr.Row():
-            lora_rank = gr.Slider(value=8, minimum=1, maximum=1024, step=1)
-            lora_dropout = gr.Slider(value=0.1, minimum=0, maximum=1, step=0.01)
-            lora_target = gr.Textbox()
-            additional_target = gr.Textbox()
+            lora_rank = gr.Slider(value=8, minimum=1, maximum=1024, step=1, scale=1)
+            lora_alpha = gr.Slider(value=16, minimum=1, maximum=2048, step=0.1, scale=1)
+            lora_dropout = gr.Slider(value=0.1, minimum=0, maximum=1, step=0.01, scale=1)
+            lora_target = gr.Textbox(scale=2)

-            with gr.Column():
-                use_rslora = gr.Checkbox()
-                create_new_adapter = gr.Checkbox()
+        with gr.Row():
+            use_rslora = gr.Checkbox(scale=1)
+            use_dora = gr.Checkbox(scale=1)
+            create_new_adapter = gr.Checkbox(scale=1)
+            additional_target = gr.Textbox(scale=2)

-    input_elems.update({lora_rank, lora_dropout, lora_target, additional_target, use_rslora, create_new_adapter})
+    input_elems.update(
+        {lora_rank, lora_alpha, lora_dropout, lora_target, use_rslora, use_dora, create_new_adapter, additional_target}
+    )
    elem_dict.update(
        dict(
            lora_tab=lora_tab,
            lora_rank=lora_rank,
+            lora_alpha=lora_alpha,
            lora_dropout=lora_dropout,
            lora_target=lora_target,
-            additional_target=additional_target,
            use_rslora=use_rslora,
+            use_dora=use_dora,
            create_new_adapter=create_new_adapter,
+            additional_target=additional_target,
        )
    )

--- a/src/llmtuner/webui/locales.py
+++ b/src/llmtuner/webui/locales.py
@@ -572,6 +572,20 @@ LOCALES = {
            "info": "LoRA 矩阵的秩。",
        },
    },
+    "lora_alpha": {
+        "en": {
+            "label": "LoRA Alpha",
+            "info": "Lora scaling coefficient.",
+        },
+        "ru": {
+            "label": "LoRA Alpha",
+            "info": "Коэффициент масштабирования LoRA.",
+        },
+        "zh": {
+            "label": "LoRA 缩放系数",
+            "info": "LoRA 缩放系数大小。",
+        },
+    },
    "lora_dropout": {
        "en": {
            "label": "LoRA Dropout",
@@ -600,6 +614,48 @@ LOCALES = {
            "info": "应用 LoRA 的目标模块名称。使用英文逗号分隔多个名称。",
        },
    },
+    "use_rslora": {
+        "en": {
+            "label": "Use rslora",
+            "info": "Use the rank stabilization scaling factor for LoRA layer.",
+        },
+        "ru": {
+            "label": "Использовать rslora",
+            "info": "Использовать коэффициент масштабирования стабилизации ранга для слоя LoRA.",
+        },
+        "zh": {
+            "label": "使用 rslora",
+            "info": "对 LoRA 层使用秩稳定缩放方法。",
+        },
+    },
+    "use_dora": {
+        "en": {
+            "label": "Use DoRA",
+            "info": "Use weight-decomposed LoRA.",
+        },
+        "ru": {
+            "label": "Используйте DoRA",
+            "info": "Используйте LoRA с декомпозицией весов.",
+        },
+        "zh": {
+            "label": "使用 DoRA",
+            "info": "使用权重分解的 LoRA。",
+        },
+    },
+    "create_new_adapter": {
+        "en": {
+            "label": "Create new adapter",
+            "info": "Create a new adapter with randomly initialized weight upon the existing one.",
+        },
+        "ru": {
+            "label": "Создать новый адаптер",
+            "info": "Создать новый адаптер с случайной инициализацией веса на основе существующего.",
+        },
+        "zh": {
+            "label": "新建适配器",
+            "info": "在现有的适配器上创建一个随机初始化后的新适配器。",
+        },
+    },
    "additional_target": {
        "en": {
            "label": "Additional modules (optional)",
@@ -617,34 +673,6 @@ LOCALES = {
            "info": "除 LoRA 层以外的可训练模块名称。使用英文逗号分隔多个名称。",
        },
    },
-    "use_rslora": {
-        "en": {
-            "label": "Use rslora",
-            "info": "Use the rank stabilization scaling factor for LoRA layer.",
-        },
-        "ru": {
-            "label": "Использовать rslora",
-            "info": "Использовать коэффициент масштабирования стабилизации ранга для слоя LoRA.",
-        },
-        "zh": {
-            "label": "使用 rslora",
-            "info": "对 LoRA 层使用秩稳定缩放方法。",
-        },
-    },
-    "create_new_adapter": {
-        "en": {
-            "label": "Create new adapter",
-            "info": "Create a new adapter with randomly initialized weight upon the existing one.",
-        },
-        "ru": {
-            "label": "Создать новый адаптер",
-            "info": "Создать новый адаптер с случайной инициализацией веса на основе существующего.",
-        },
-        "zh": {
-            "label": "新建适配器",
-            "info": "在现有的适配器上创建一个随机初始化后的新适配器。",
-        },
-    },
    "rlhf_tab": {
        "en": {
            "label": "RLHF configurations",
@@ -1055,6 +1083,11 @@ ALERTS = {
        "ru": "Неверная схема JSON.",
        "zh": "Json 格式错误。",
    },
+    "warn_no_cuda": {
+        "en": "CUDA environment was not detected.",
+        "ru": "Среда CUDA не обнаружена.",
+        "zh": "未检测到 CUDA 环境。",
+    },
    "info_aborting": {
        "en": "Aborted, wait for terminating...",
        "ru": "Прервано, ожидание завершения...",
--- a/src/llmtuner/webui/runner.py
+++ b/src/llmtuner/webui/runner.py
@@ -8,6 +8,7 @@ import gradio as gr
 import transformers
 from gradio.components import Component  # cannot use TYPE_CHECKING here
 from transformers.trainer import TRAINING_ARGS_NAME
+from transformers.utils import is_torch_cuda_available

 from ..extras.callbacks import LogCallback
 from ..extras.constants import TRAINING_STAGES
@@ -64,12 +65,15 @@ class Runner:
        if len(dataset) == 0:
            return ALERTS["err_no_dataset"][lang]

-        if self.demo_mode and (not from_preview):
+        if not from_preview and self.demo_mode:
            return ALERTS["err_demo"][lang]

        if not from_preview and get_device_count() > 1:
            return ALERTS["err_device_count"][lang]

+        if not from_preview and not is_torch_cuda_available():
+            gr.Warning(ALERTS["warn_no_cuda"][lang])
+
        self.aborted = False
        self.logger_handler.reset()
        self.trainer_callback = LogCallback(self)
@@ -139,11 +143,13 @@ class Runner:
            args["num_layer_trainable"] = int(get("train.num_layer_trainable"))
            args["name_module_trainable"] = get("train.name_module_trainable")
        elif args["finetuning_type"] == "lora":
-            args["lora_rank"] = get("train.lora_rank")
-            args["lora_dropout"] = get("train.lora_dropout")
+            args["lora_rank"] = int(get("train.lora_rank"))
+            args["lora_alpha"] = float(get("train.lora_alpha"))
+            args["lora_dropout"] = float(get("train.lora_dropout"))
            args["lora_target"] = get("train.lora_target") or get_module(get("top.model_name"))
-            args["additional_target"] = get("train.additional_target") or None
            args["use_rslora"] = get("train.use_rslora")
+            args["use_dora"] = get("train.use_dora")
+            args["additional_target"] = get("train.additional_target") or None
            if args["stage"] in ["rm", "ppo", "dpo"]:
                args["create_new_adapter"] = args["quantization_bit"] is None
            else:
--- a/src/llmtuner/webui/utils.py
+++ b/src/llmtuner/webui/utils.py
@@ -44,11 +44,14 @@ def can_quantize(finetuning_type: str) -> Dict[str, Any]:
 def check_json_schema(text: str, lang: str) -> None:
    try:
        tools = json.loads(text)
-        for tool in tools:
-            assert "name" in tool
-    except AssertionError:
+        if tools:
+            assert isinstance(tools, list)
+            for tool in tools:
+                if "name" not in tool:
+                    raise ValueError("Name not found.")
+    except ValueError:
        gr.Warning(ALERTS["err_tool_name"][lang])
-    except json.JSONDecodeError:
+    except Exception:
        gr.Warning(ALERTS["err_json_schema"][lang])